diff --git "a/README.md" "b/README.md" --- "a/README.md" +++ "b/README.md" @@ -19,657 +19,22061 @@ tags: - granite - embeddings - multilingual +- mteb model-index: - name: ibm-granite/granite-embedding-278m-multilingual results: - dataset: - type: miracl/mmteb-miracl - name: Miracl (en) + config: en-ext + name: MTEB AmazonCounterfactualClassification (en-ext) + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + split: test + type: mteb/amazon_counterfactual + metrics: + - type: accuracy + value: 73.4333 + - type: f1 + value: 61.2301 + - type: f1_weighted + value: 78.40899999999999 + - type: ap + value: 23.347 + - type: ap_weighted + value: 23.347 + - type: main_score + value: 73.4333 + task: + type: Classification + - dataset: config: en - split: dev + name: MTEB AmazonCounterfactualClassification (en) + revision: e8379541af4e31359cca9fbcf4b00f2671dba205 + split: test + type: mteb/amazon_counterfactual + metrics: + - type: accuracy + value: 71.806 + - type: f1 + value: 65.6467 + - type: f1_weighted + value: 74.4815 + - type: ap + value: 34.045700000000004 + - type: ap_weighted + value: 34.045700000000004 + - type: main_score + value: 71.806 task: - type: Retrieval + type: Classification + - dataset: + config: default + name: MTEB AmazonPolarityClassification (default) + revision: e2d317d38cd51312af73b3d32a06d1a08b442046 + split: test + type: mteb/amazon_polarity + metrics: + - type: accuracy + value: 67.5907 + - type: f1 + value: 67.36370000000001 + - type: f1_weighted + value: 67.36370000000001 + - type: ap + value: 62.0368 + - type: ap_weighted + value: 62.0368 + - type: main_score + value: 67.5907 + task: + type: Classification + - dataset: + config: en + name: MTEB AmazonReviewsClassification (en) + revision: 1399c76144fd37290681b995c656ef9b2e06e26d + split: test + type: mteb/amazon_reviews_multi + metrics: + - type: accuracy + value: 37.278 + - type: f1 + value: 36.4099 + - type: f1_weighted + value: 36.4099 + - type: main_score + value: 37.278 + task: + type: Classification + - dataset: + config: default + name: MTEB AppsRetrieval (default) + revision: f22508f96b7a36c2415181ed8bb76f76e04ae2d5 + split: test + type: CoIR-Retrieval/apps metrics: - type: ndcg_at_1 - value: 0.45557 + value: 3.453 + - type: ndcg_at_3 + value: 4.882000000000001 + - type: ndcg_at_5 + value: 5.564 - type: ndcg_at_10 - value: 0.49372 + value: 6.214 + - type: ndcg_at_20 + value: 6.814000000000001 - type: ndcg_at_100 - value: 0.5728 + value: 8.581 - type: ndcg_at_1000 - value: 0.59187 - - type: ndcg_at_20 - value: 0.52863 - - type: ndcg_at_3 - value: 0.43969 - - type: ndcg_at_5 - value: 0.45551 + value: 12.215 + - type: map_at_1 + value: 3.453 + - type: map_at_3 + value: 4.515000000000001 + - type: map_at_5 + value: 4.89 + - type: map_at_10 + value: 5.151 + - type: map_at_20 + value: 5.313 + - type: map_at_100 + value: 5.539000000000001 + - type: map_at_1000 + value: 5.638 - type: recall_at_1 - value: 0.21785 + value: 3.453 + - type: recall_at_3 + value: 5.949999999999999 + - type: recall_at_5 + value: 7.623 - type: recall_at_10 - value: 0.59513 + value: 9.668000000000001 + - type: recall_at_20 + value: 12.058 - type: recall_at_100 - value: 0.85785 + value: 21.859 - type: recall_at_1000 - value: 0.96041 - - type: recall_at_20 - value: 0.69357 - - type: recall_at_3 - value: 0.40403 - - type: recall_at_5 - value: 0.48499 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (ar) - config: ar - split: dev + value: 52.722 + - type: precision_at_1 + value: 3.453 + - type: precision_at_3 + value: 1.983 + - type: precision_at_5 + value: 1.525 + - type: precision_at_10 + value: 0.967 + - type: precision_at_20 + value: 0.603 + - type: precision_at_100 + value: 0.219 + - type: precision_at_1000 + value: 0.053 + - type: mrr_at_1 + value: 3.4528999999999996 + - type: mrr_at_3 + value: 4.5153 + - type: mrr_at_5 + value: 4.889799999999999 + - type: mrr_at_10 + value: 5.1507 + - type: mrr_at_20 + value: 5.3135 + - type: mrr_at_100 + value: 5.5391 + - type: mrr_at_1000 + value: 5.6382 + - type: nauc_ndcg_at_1_max + value: 37.1714 + - type: nauc_ndcg_at_1_std + value: 15.306700000000001 + - type: nauc_ndcg_at_1_diff1 + value: 46.2252 + - type: nauc_ndcg_at_3_max + value: 32.0309 + - type: nauc_ndcg_at_3_std + value: 14.2983 + - type: nauc_ndcg_at_3_diff1 + value: 34.7174 + - type: nauc_ndcg_at_5_max + value: 29.3613 + - type: nauc_ndcg_at_5_std + value: 13.0358 + - type: nauc_ndcg_at_5_diff1 + value: 30.8369 + - type: nauc_ndcg_at_10_max + value: 26.820100000000004 + - type: nauc_ndcg_at_10_std + value: 12.3422 + - type: nauc_ndcg_at_10_diff1 + value: 27.3719 + - type: nauc_ndcg_at_20_max + value: 25.5643 + - type: nauc_ndcg_at_20_std + value: 11.383000000000001 + - type: nauc_ndcg_at_20_diff1 + value: 25.7058 + - type: nauc_ndcg_at_100_max + value: 23.2131 + - type: nauc_ndcg_at_100_std + value: 12.4787 + - type: nauc_ndcg_at_100_diff1 + value: 21.6874 + - type: nauc_ndcg_at_1000_max + value: 22.900499999999997 + - type: nauc_ndcg_at_1000_std + value: 13.2218 + - type: nauc_ndcg_at_1000_diff1 + value: 19.668 + - type: nauc_map_at_1_max + value: 37.1714 + - type: nauc_map_at_1_std + value: 15.306700000000001 + - type: nauc_map_at_1_diff1 + value: 46.2252 + - type: nauc_map_at_3_max + value: 33.1012 + - type: nauc_map_at_3_std + value: 14.4117 + - type: nauc_map_at_3_diff1 + value: 36.8859 + - type: nauc_map_at_5_max + value: 31.404700000000002 + - type: nauc_map_at_5_std + value: 13.5956 + - type: nauc_map_at_5_diff1 + value: 34.3454 + - type: nauc_map_at_10_max + value: 30.1013 + - type: nauc_map_at_10_std + value: 13.2253 + - type: nauc_map_at_10_diff1 + value: 32.487 + - type: nauc_map_at_20_max + value: 29.5747 + - type: nauc_map_at_20_std + value: 12.843499999999999 + - type: nauc_map_at_20_diff1 + value: 31.8252 + - type: nauc_map_at_100_max + value: 28.968899999999998 + - type: nauc_map_at_100_std + value: 12.967699999999999 + - type: nauc_map_at_100_diff1 + value: 30.924000000000003 + - type: nauc_map_at_1000_max + value: 28.894599999999997 + - type: nauc_map_at_1000_std + value: 12.997800000000002 + - type: nauc_map_at_1000_diff1 + value: 30.7653 + - type: nauc_recall_at_1_max + value: 37.1714 + - type: nauc_recall_at_1_std + value: 15.306700000000001 + - type: nauc_recall_at_1_diff1 + value: 46.2252 + - type: nauc_recall_at_3_max + value: 29.6485 + - type: nauc_recall_at_3_std + value: 14.072799999999999 + - type: nauc_recall_at_3_diff1 + value: 29.9536 + - type: nauc_recall_at_5_max + value: 25.251099999999997 + - type: nauc_recall_at_5_std + value: 11.9121 + - type: nauc_recall_at_5_diff1 + value: 23.9203 + - type: nauc_recall_at_10_max + value: 20.8856 + - type: nauc_recall_at_10_std + value: 10.7653 + - type: nauc_recall_at_10_diff1 + value: 18.3716 + - type: nauc_recall_at_20_max + value: 18.9378 + - type: nauc_recall_at_20_std + value: 8.8933 + - type: nauc_recall_at_20_diff1 + value: 15.7693 + - type: nauc_recall_at_100_max + value: 15.7027 + - type: nauc_recall_at_100_std + value: 12.6519 + - type: nauc_recall_at_100_diff1 + value: 9.2726 + - type: nauc_recall_at_1000_max + value: 16.2321 + - type: nauc_recall_at_1000_std + value: 15.2717 + - type: nauc_recall_at_1000_diff1 + value: 4.4337 + - type: nauc_precision_at_1_max + value: 37.1714 + - type: nauc_precision_at_1_std + value: 15.306700000000001 + - type: nauc_precision_at_1_diff1 + value: 46.2252 + - type: nauc_precision_at_3_max + value: 29.6485 + - type: nauc_precision_at_3_std + value: 14.072799999999999 + - type: nauc_precision_at_3_diff1 + value: 29.9536 + - type: nauc_precision_at_5_max + value: 25.251099999999997 + - type: nauc_precision_at_5_std + value: 11.9121 + - type: nauc_precision_at_5_diff1 + value: 23.9203 + - type: nauc_precision_at_10_max + value: 20.8856 + - type: nauc_precision_at_10_std + value: 10.7653 + - type: nauc_precision_at_10_diff1 + value: 18.3716 + - type: nauc_precision_at_20_max + value: 18.9378 + - type: nauc_precision_at_20_std + value: 8.8933 + - type: nauc_precision_at_20_diff1 + value: 15.7693 + - type: nauc_precision_at_100_max + value: 15.7027 + - type: nauc_precision_at_100_std + value: 12.6519 + - type: nauc_precision_at_100_diff1 + value: 9.2726 + - type: nauc_precision_at_1000_max + value: 16.2321 + - type: nauc_precision_at_1000_std + value: 15.2717 + - type: nauc_precision_at_1000_diff1 + value: 4.4337 + - type: nauc_mrr_at_1_max + value: 37.1714 + - type: nauc_mrr_at_1_std + value: 15.306700000000001 + - type: nauc_mrr_at_1_diff1 + value: 46.2252 + - type: nauc_mrr_at_3_max + value: 33.1012 + - type: nauc_mrr_at_3_std + value: 14.4117 + - type: nauc_mrr_at_3_diff1 + value: 36.8859 + - type: nauc_mrr_at_5_max + value: 31.404700000000002 + - type: nauc_mrr_at_5_std + value: 13.5956 + - type: nauc_mrr_at_5_diff1 + value: 34.3454 + - type: nauc_mrr_at_10_max + value: 30.1013 + - type: nauc_mrr_at_10_std + value: 13.2253 + - type: nauc_mrr_at_10_diff1 + value: 32.487 + - type: nauc_mrr_at_20_max + value: 29.5747 + - type: nauc_mrr_at_20_std + value: 12.843499999999999 + - type: nauc_mrr_at_20_diff1 + value: 31.8252 + - type: nauc_mrr_at_100_max + value: 28.968899999999998 + - type: nauc_mrr_at_100_std + value: 12.967699999999999 + - type: nauc_mrr_at_100_diff1 + value: 30.9239 + - type: nauc_mrr_at_1000_max + value: 28.894599999999997 + - type: nauc_mrr_at_1000_std + value: 12.997800000000002 + - type: nauc_mrr_at_1000_diff1 + value: 30.7653 + - type: main_score + value: 6.214 task: type: Retrieval + - dataset: + config: default + name: MTEB ArguAna (default) + revision: c22ab2a51041ffd869aaddef7af8d8215647e41a + split: test + type: mteb/arguana metrics: - type: ndcg_at_1 - value: 0.57459 + value: 31.152 + - type: ndcg_at_3 + value: 45.050000000000004 + - type: ndcg_at_5 + value: 50.458999999999996 - type: ndcg_at_10 - value: 0.64238 + value: 55.24400000000001 + - type: ndcg_at_20 + value: 57.918000000000006 - type: ndcg_at_100 - value: 0.6867 + value: 58.97 - type: ndcg_at_1000 - value: 0.6951 - - type: ndcg_at_20 - value: 0.66455 - - type: ndcg_at_3 - value: 0.58162 - - type: ndcg_at_5 - value: 0.60831 + value: 59.080999999999996 + - type: map_at_1 + value: 31.152 + - type: map_at_3 + value: 41.513 + - type: map_at_5 + value: 44.542 + - type: map_at_10 + value: 46.544000000000004 + - type: map_at_20 + value: 47.304 + - type: map_at_100 + value: 47.467999999999996 + - type: map_at_1000 + value: 47.473 - type: recall_at_1 - value: 0.38064 + value: 31.152 + - type: recall_at_3 + value: 55.334 + - type: recall_at_5 + value: 68.35 - type: recall_at_10 - value: 0.75098 + value: 83.001 + - type: recall_at_20 + value: 93.38499999999999 - type: recall_at_100 - value: 0.91203 + value: 98.791 - type: recall_at_1000 - value: 0.96706 - - type: recall_at_20 - value: 0.81978 - - type: recall_at_3 - value: 0.58618 - - type: recall_at_5 - value: 0.66353 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (bn) - config: bn - split: dev + value: 99.644 + - type: precision_at_1 + value: 31.152 + - type: precision_at_3 + value: 18.445 + - type: precision_at_5 + value: 13.669999999999998 + - type: precision_at_10 + value: 8.3 + - type: precision_at_20 + value: 4.6690000000000005 + - type: precision_at_100 + value: 0.988 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 31.7212 + - type: mrr_at_3 + value: 41.7141 + - type: mrr_at_5 + value: 44.754599999999996 + - type: mrr_at_10 + value: 46.7491 + - type: mrr_at_20 + value: 47.515299999999996 + - type: mrr_at_100 + value: 47.679300000000005 + - type: mrr_at_1000 + value: 47.6841 + - type: nauc_ndcg_at_1_max + value: -7.8191 + - type: nauc_ndcg_at_1_std + value: -4.0581 + - type: nauc_ndcg_at_1_diff1 + value: 14.383199999999999 + - type: nauc_ndcg_at_3_max + value: -4.6856 + - type: nauc_ndcg_at_3_std + value: -3.4165 + - type: nauc_ndcg_at_3_diff1 + value: 10.7764 + - type: nauc_ndcg_at_5_max + value: -3.2999 + - type: nauc_ndcg_at_5_std + value: -3.6675 + - type: nauc_ndcg_at_5_diff1 + value: 11.6249 + - type: nauc_ndcg_at_10_max + value: -3.2984 + - type: nauc_ndcg_at_10_std + value: -3.0373 + - type: nauc_ndcg_at_10_diff1 + value: 11.9938 + - type: nauc_ndcg_at_20_max + value: -3.147 + - type: nauc_ndcg_at_20_std + value: -2.9219 + - type: nauc_ndcg_at_20_diff1 + value: 12.4893 + - type: nauc_ndcg_at_100_max + value: -4.2572 + - type: nauc_ndcg_at_100_std + value: -2.8537 + - type: nauc_ndcg_at_100_diff1 + value: 12.1039 + - type: nauc_ndcg_at_1000_max + value: -4.3526 + - type: nauc_ndcg_at_1000_std + value: -3.0145 + - type: nauc_ndcg_at_1000_diff1 + value: 12.1685 + - type: nauc_map_at_1_max + value: -7.8191 + - type: nauc_map_at_1_std + value: -4.0581 + - type: nauc_map_at_1_diff1 + value: 14.383199999999999 + - type: nauc_map_at_3_max + value: -5.5556 + - type: nauc_map_at_3_std + value: -3.515 + - type: nauc_map_at_3_diff1 + value: 11.5486 + - type: nauc_map_at_5_max + value: -4.840599999999999 + - type: nauc_map_at_5_std + value: -3.6663 + - type: nauc_map_at_5_diff1 + value: 12.053899999999999 + - type: nauc_map_at_10_max + value: -4.9401 + - type: nauc_map_at_10_std + value: -3.3724 + - type: nauc_map_at_10_diff1 + value: 12.1558 + - type: nauc_map_at_20_max + value: -4.9365 + - type: nauc_map_at_20_std + value: -3.3676999999999997 + - type: nauc_map_at_20_diff1 + value: 12.2729 + - type: nauc_map_at_100_max + value: -5.0695 + - type: nauc_map_at_100_std + value: -3.3561 + - type: nauc_map_at_100_diff1 + value: 12.237 + - type: nauc_map_at_1000_max + value: -5.0709 + - type: nauc_map_at_1000_std + value: -3.3594 + - type: nauc_map_at_1000_diff1 + value: 12.2408 + - type: nauc_recall_at_1_max + value: -7.8191 + - type: nauc_recall_at_1_std + value: -4.0581 + - type: nauc_recall_at_1_diff1 + value: 14.383199999999999 + - type: nauc_recall_at_3_max + value: -2.0358 + - type: nauc_recall_at_3_std + value: -3.1464 + - type: nauc_recall_at_3_diff1 + value: 8.510900000000001 + - type: nauc_recall_at_5_max + value: 2.4358999999999997 + - type: nauc_recall_at_5_std + value: -3.727 + - type: nauc_recall_at_5_diff1 + value: 10.2867 + - type: nauc_recall_at_10_max + value: 6.5777 + - type: nauc_recall_at_10_std + value: -1.0198 + - type: nauc_recall_at_10_diff1 + value: 11.9244 + - type: nauc_recall_at_20_max + value: 22.8541 + - type: nauc_recall_at_20_std + value: 4.1539 + - type: nauc_recall_at_20_diff1 + value: 19.3648 + - type: nauc_recall_at_100_max + value: 18.5148 + - type: nauc_recall_at_100_std + value: 41.1822 + - type: nauc_recall_at_100_diff1 + value: 5.1883 + - type: nauc_recall_at_1000_max + value: 13.995099999999999 + - type: nauc_recall_at_1000_std + value: 53.7961 + - type: nauc_recall_at_1000_diff1 + value: 14.8451 + - type: nauc_precision_at_1_max + value: -7.8191 + - type: nauc_precision_at_1_std + value: -4.0581 + - type: nauc_precision_at_1_diff1 + value: 14.383199999999999 + - type: nauc_precision_at_3_max + value: -2.0358 + - type: nauc_precision_at_3_std + value: -3.1464 + - type: nauc_precision_at_3_diff1 + value: 8.510900000000001 + - type: nauc_precision_at_5_max + value: 2.4358999999999997 + - type: nauc_precision_at_5_std + value: -3.727 + - type: nauc_precision_at_5_diff1 + value: 10.2867 + - type: nauc_precision_at_10_max + value: 6.5777 + - type: nauc_precision_at_10_std + value: -1.0198 + - type: nauc_precision_at_10_diff1 + value: 11.9244 + - type: nauc_precision_at_20_max + value: 22.8541 + - type: nauc_precision_at_20_std + value: 4.1539 + - type: nauc_precision_at_20_diff1 + value: 19.3648 + - type: nauc_precision_at_100_max + value: 18.5148 + - type: nauc_precision_at_100_std + value: 41.1822 + - type: nauc_precision_at_100_diff1 + value: 5.1883 + - type: nauc_precision_at_1000_max + value: 13.995099999999999 + - type: nauc_precision_at_1000_std + value: 53.7961 + - type: nauc_precision_at_1000_diff1 + value: 14.8451 + - type: nauc_mrr_at_1_max + value: -8.1904 + - type: nauc_mrr_at_1_std + value: -4.0896 + - type: nauc_mrr_at_1_diff1 + value: 12.7103 + - type: nauc_mrr_at_3_max + value: -6.6608 + - type: nauc_mrr_at_3_std + value: -3.6741 + - type: nauc_mrr_at_3_diff1 + value: 9.851 + - type: nauc_mrr_at_5_max + value: -5.7596 + - type: nauc_mrr_at_5_std + value: -3.7391 + - type: nauc_mrr_at_5_diff1 + value: 10.4908 + - type: nauc_mrr_at_10_max + value: -5.8613 + - type: nauc_mrr_at_10_std + value: -3.4377999999999997 + - type: nauc_mrr_at_10_diff1 + value: 10.5641 + - type: nauc_mrr_at_20_max + value: -5.8497 + - type: nauc_mrr_at_20_std + value: -3.4543 + - type: nauc_mrr_at_20_diff1 + value: 10.6822 + - type: nauc_mrr_at_100_max + value: -5.9873 + - type: nauc_mrr_at_100_std + value: -3.4431000000000003 + - type: nauc_mrr_at_100_diff1 + value: 10.6379 + - type: nauc_mrr_at_1000_max + value: -5.9887999999999995 + - type: nauc_mrr_at_1000_std + value: -3.4465000000000003 + - type: nauc_mrr_at_1000_diff1 + value: 10.641399999999999 + - type: main_score + value: 55.24400000000001 task: type: Retrieval + - dataset: + config: default + name: MTEB ArxivClusteringP2P (default) + revision: a122ad7f3f0291bf49cc6f4d32aa80929df69d5d + split: test + type: mteb/arxiv-clustering-p2p + metrics: + - type: v_measure + value: 43.1321 + - type: v_measure_std + value: 13.594000000000001 + - type: main_score + value: 43.1321 + task: + type: Clustering + - dataset: + config: default + name: MTEB ArxivClusteringS2S (default) + revision: f910caf1a6075f7329cdf8c1a6135696f37dbd53 + split: test + type: mteb/arxiv-clustering-s2s + metrics: + - type: v_measure + value: 32.9343 + - type: v_measure_std + value: 14.2478 + - type: main_score + value: 32.9343 + task: + type: Clustering + - dataset: + config: default + name: MTEB AskUbuntuDupQuestions (default) + revision: 2000358ca161889fa9c082cb41daa8dcfb161a54 + split: test + type: mteb/askubuntudupquestions-reranking + metrics: + - type: map + value: 62.3443 + - type: mrr + value: 76.3882 + - type: nAUC_map_max + value: 28.3073 + - type: nAUC_map_std + value: 15.5307 + - type: nAUC_map_diff1 + value: 12.6855 + - type: nAUC_mrr_max + value: 36.409200000000006 + - type: nAUC_mrr_std + value: 22.6271 + - type: nAUC_mrr_diff1 + value: 19.1211 + - type: main_score + value: 62.3443 + task: + type: Reranking + - dataset: + config: default + name: MTEB BIOSSES (default) + revision: d3fb88f8f02e40887cd149695127462bbcf29b4a + split: test + type: mteb/biosses-sts + metrics: + - type: pearson + value: 84.3253 + - type: spearman + value: 81.6362 + - type: cosine_pearson + value: 84.3253 + - type: cosine_spearman + value: 81.6362 + - type: manhattan_pearson + value: 82.70960000000001 + - type: manhattan_spearman + value: 81.3037 + - type: euclidean_pearson + value: 82.6906 + - type: euclidean_spearman + value: 81.6362 + - type: main_score + value: 81.6362 + task: + type: STS + - dataset: + config: default + name: MTEB Banking77Classification (default) + revision: 0fd18e25b25c072e09e0d92ab615fda904d66300 + split: test + type: mteb/banking77 + metrics: + - type: accuracy + value: 78.0617 + - type: f1 + value: 77.2085 + - type: f1_weighted + value: 77.2085 + - type: main_score + value: 78.0617 + task: + type: Classification + - dataset: + config: default + name: MTEB BiorxivClusteringP2P (default) + revision: 65b79d1d13f80053f67aca9498d9402c2d9f1f40 + split: test + type: mteb/biorxiv-clustering-p2p + metrics: + - type: v_measure + value: 35.8271 + - type: v_measure_std + value: 0.7191000000000001 + - type: main_score + value: 35.8271 + task: + type: Clustering + - dataset: + config: default + name: MTEB BiorxivClusteringS2S (default) + revision: 258694dd0231531bc1fd9de6ceb52a0853c6d908 + split: test + type: mteb/biorxiv-clustering-s2s + metrics: + - type: v_measure + value: 30.3905 + - type: v_measure_std + value: 0.7136 + - type: main_score + value: 30.3905 + task: + type: Clustering + - dataset: + config: python + name: MTEB COIRCodeSearchNetRetrieval (python) + revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 + split: test + type: CoIR-Retrieval/CodeSearchNet metrics: - type: ndcg_at_1 - value: 0.60341 + value: 83.22800000000001 + - type: ndcg_at_3 + value: 87.41799999999999 + - type: ndcg_at_5 + value: 88.089 - type: ndcg_at_10 - value: 0.68055 + value: 88.789 + - type: ndcg_at_20 + value: 89.156 - type: ndcg_at_100 - value: 0.72008 + value: 89.60900000000001 - type: ndcg_at_1000 - value: 0.72716 - - type: ndcg_at_20 - value: 0.69914 - - type: ndcg_at_3 - value: 0.60805 - - type: ndcg_at_5 - value: 0.64486 + value: 89.79 + - type: map_at_1 + value: 83.22800000000001 + - type: map_at_3 + value: 86.431 + - type: map_at_5 + value: 86.80499999999999 + - type: map_at_10 + value: 87.09599999999999 + - type: map_at_20 + value: 87.198 + - type: map_at_100 + value: 87.263 + - type: map_at_1000 + value: 87.27000000000001 - type: recall_at_1 - value: 0.37948 + value: 83.22800000000001 + - type: recall_at_3 + value: 90.253 + - type: recall_at_5 + value: 91.876 - type: recall_at_10 - value: 0.80609 + value: 94.03399999999999 + - type: recall_at_20 + value: 95.475 - type: recall_at_100 - value: 0.94305 + value: 97.882 - type: recall_at_1000 - value: 0.98625 - - type: recall_at_20 - value: 0.86141 - - type: recall_at_3 - value: 0.61095 - - type: recall_at_5 - value: 0.71316 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (de) - config: de - split: dev + value: 99.316 + - type: precision_at_1 + value: 83.22800000000001 + - type: precision_at_3 + value: 30.084 + - type: precision_at_5 + value: 18.375 + - type: precision_at_10 + value: 9.403 + - type: precision_at_20 + value: 4.774 + - type: precision_at_100 + value: 0.979 + - type: precision_at_1000 + value: 0.099 + - type: mrr_at_1 + value: 83.235 + - type: mrr_at_3 + value: 86.4336 + - type: mrr_at_5 + value: 86.8077 + - type: mrr_at_10 + value: 87.0979 + - type: mrr_at_20 + value: 87.2001 + - type: mrr_at_100 + value: 87.26509999999999 + - type: mrr_at_1000 + value: 87.2718 + - type: nauc_ndcg_at_1_max + value: 82.2462 + - type: nauc_ndcg_at_1_std + value: 11.4635 + - type: nauc_ndcg_at_1_diff1 + value: 90.5106 + - type: nauc_ndcg_at_3_max + value: 83.9742 + - type: nauc_ndcg_at_3_std + value: 12.7085 + - type: nauc_ndcg_at_3_diff1 + value: 88.2182 + - type: nauc_ndcg_at_5_max + value: 84.18870000000001 + - type: nauc_ndcg_at_5_std + value: 13.167499999999999 + - type: nauc_ndcg_at_5_diff1 + value: 88.44999999999999 + - type: nauc_ndcg_at_10_max + value: 84.2219 + - type: nauc_ndcg_at_10_std + value: 13.5219 + - type: nauc_ndcg_at_10_diff1 + value: 88.6386 + - type: nauc_ndcg_at_20_max + value: 84.2289 + - type: nauc_ndcg_at_20_std + value: 14.0686 + - type: nauc_ndcg_at_20_diff1 + value: 88.7516 + - type: nauc_ndcg_at_100_max + value: 84.12049999999999 + - type: nauc_ndcg_at_100_std + value: 14.1778 + - type: nauc_ndcg_at_100_diff1 + value: 88.8592 + - type: nauc_ndcg_at_1000_max + value: 84.0367 + - type: nauc_ndcg_at_1000_std + value: 13.9125 + - type: nauc_ndcg_at_1000_diff1 + value: 88.9054 + - type: nauc_map_at_1_max + value: 82.2462 + - type: nauc_map_at_1_std + value: 11.4635 + - type: nauc_map_at_1_diff1 + value: 90.5106 + - type: nauc_map_at_3_max + value: 83.5638 + - type: nauc_map_at_3_std + value: 12.3576 + - type: nauc_map_at_3_diff1 + value: 88.8502 + - type: nauc_map_at_5_max + value: 83.6625 + - type: nauc_map_at_5_std + value: 12.582099999999999 + - type: nauc_map_at_5_diff1 + value: 88.9876 + - type: nauc_map_at_10_max + value: 83.6605 + - type: nauc_map_at_10_std + value: 12.6859 + - type: nauc_map_at_10_diff1 + value: 89.07119999999999 + - type: nauc_map_at_20_max + value: 83.65629999999999 + - type: nauc_map_at_20_std + value: 12.8105 + - type: nauc_map_at_20_diff1 + value: 89.1036 + - type: nauc_map_at_100_max + value: 83.6413 + - type: nauc_map_at_100_std + value: 12.823699999999999 + - type: nauc_map_at_100_diff1 + value: 89.1193 + - type: nauc_map_at_1000_max + value: 83.6386 + - type: nauc_map_at_1000_std + value: 12.815999999999999 + - type: nauc_map_at_1000_diff1 + value: 89.1209 + - type: nauc_recall_at_1_max + value: 82.2462 + - type: nauc_recall_at_1_std + value: 11.4635 + - type: nauc_recall_at_1_diff1 + value: 90.5106 + - type: nauc_recall_at_3_max + value: 85.512 + - type: nauc_recall_at_3_std + value: 14.061399999999999 + - type: nauc_recall_at_3_diff1 + value: 85.7898 + - type: nauc_recall_at_5_max + value: 86.5434 + - type: nauc_recall_at_5_std + value: 15.894400000000001 + - type: nauc_recall_at_5_diff1 + value: 86.0934 + - type: nauc_recall_at_10_max + value: 87.59909999999999 + - type: nauc_recall_at_10_std + value: 18.9872 + - type: nauc_recall_at_10_diff1 + value: 86.26740000000001 + - type: nauc_recall_at_20_max + value: 88.76190000000001 + - type: nauc_recall_at_20_std + value: 25.6618 + - type: nauc_recall_at_20_diff1 + value: 86.5002 + - type: nauc_recall_at_100_max + value: 91.0976 + - type: nauc_recall_at_100_std + value: 40.9161 + - type: nauc_recall_at_100_diff1 + value: 86.5441 + - type: nauc_recall_at_1000_max + value: 96.018 + - type: nauc_recall_at_1000_std + value: 65.6217 + - type: nauc_recall_at_1000_diff1 + value: 86.8456 + - type: nauc_precision_at_1_max + value: 82.2462 + - type: nauc_precision_at_1_std + value: 11.4635 + - type: nauc_precision_at_1_diff1 + value: 90.5106 + - type: nauc_precision_at_3_max + value: 85.512 + - type: nauc_precision_at_3_std + value: 14.061399999999999 + - type: nauc_precision_at_3_diff1 + value: 85.7898 + - type: nauc_precision_at_5_max + value: 86.5434 + - type: nauc_precision_at_5_std + value: 15.894400000000001 + - type: nauc_precision_at_5_diff1 + value: 86.0934 + - type: nauc_precision_at_10_max + value: 87.59909999999999 + - type: nauc_precision_at_10_std + value: 18.9872 + - type: nauc_precision_at_10_diff1 + value: 86.26740000000001 + - type: nauc_precision_at_20_max + value: 88.76190000000001 + - type: nauc_precision_at_20_std + value: 25.6618 + - type: nauc_precision_at_20_diff1 + value: 86.5002 + - type: nauc_precision_at_100_max + value: 91.0976 + - type: nauc_precision_at_100_std + value: 40.9161 + - type: nauc_precision_at_100_diff1 + value: 86.5441 + - type: nauc_precision_at_1000_max + value: 96.018 + - type: nauc_precision_at_1000_std + value: 65.6217 + - type: nauc_precision_at_1000_diff1 + value: 86.8456 + - type: nauc_mrr_at_1_max + value: 82.2393 + - type: nauc_mrr_at_1_std + value: 11.5163 + - type: nauc_mrr_at_1_diff1 + value: 90.50160000000001 + - type: nauc_mrr_at_3_max + value: 83.5623 + - type: nauc_mrr_at_3_std + value: 12.395 + - type: nauc_mrr_at_3_diff1 + value: 88.8463 + - type: nauc_mrr_at_5_max + value: 83.6609 + - type: nauc_mrr_at_5_std + value: 12.620700000000001 + - type: nauc_mrr_at_5_diff1 + value: 88.9836 + - type: nauc_mrr_at_10_max + value: 83.6589 + - type: nauc_mrr_at_10_std + value: 12.7255 + - type: nauc_mrr_at_10_diff1 + value: 89.0672 + - type: nauc_mrr_at_20_max + value: 83.6546 + - type: nauc_mrr_at_20_std + value: 12.8504 + - type: nauc_mrr_at_20_diff1 + value: 89.09949999999999 + - type: nauc_mrr_at_100_max + value: 83.6396 + - type: nauc_mrr_at_100_std + value: 12.8638 + - type: nauc_mrr_at_100_diff1 + value: 89.1152 + - type: nauc_mrr_at_1000_max + value: 83.6369 + - type: nauc_mrr_at_1000_std + value: 12.856100000000001 + - type: nauc_mrr_at_1000_diff1 + value: 89.1168 + - type: main_score + value: 88.789 task: type: Retrieval + - dataset: + config: javascript + name: MTEB COIRCodeSearchNetRetrieval (javascript) + revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 + split: test + type: CoIR-Retrieval/CodeSearchNet metrics: - type: ndcg_at_1 - value: 0.45574 + value: 29.14 + - type: ndcg_at_3 + value: 35.185 + - type: ndcg_at_5 + value: 37.013 - type: ndcg_at_10 - value: 0.48123 + value: 38.778 + - type: ndcg_at_20 + value: 40.184999999999995 - type: ndcg_at_100 - value: 0.56049 + value: 42.394999999999996 - type: ndcg_at_1000 - value: 0.57979 - - type: ndcg_at_20 - value: 0.51785 - - type: ndcg_at_3 - value: 0.41243 - - type: ndcg_at_5 - value: 0.4386 + value: 44.243 + - type: map_at_1 + value: 29.14 + - type: map_at_3 + value: 33.703 + - type: map_at_5 + value: 34.717999999999996 + - type: map_at_10 + value: 35.443999999999996 + - type: map_at_20 + value: 35.831 + - type: map_at_100 + value: 36.132999999999996 + - type: map_at_1000 + value: 36.193999999999996 - type: recall_at_1 - value: 0.20401 + value: 29.14 + - type: recall_at_3 + value: 39.471000000000004 + - type: recall_at_5 + value: 43.908 - type: recall_at_10 - value: 0.58779 + value: 49.376999999999995 + - type: recall_at_20 + value: 54.937999999999995 - type: recall_at_100 - value: 0.8584 + value: 66.91 - type: recall_at_1000 - value: 0.97364 - - type: recall_at_20 - value: 0.69061 - - type: recall_at_3 - value: 0.36573 - - type: recall_at_5 - value: 0.47495 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (es) - config: es - split: dev + value: 81.98100000000001 + - type: precision_at_1 + value: 29.14 + - type: precision_at_3 + value: 13.157 + - type: precision_at_5 + value: 8.782 + - type: precision_at_10 + value: 4.938 + - type: precision_at_20 + value: 2.7470000000000003 + - type: precision_at_100 + value: 0.6689999999999999 + - type: precision_at_1000 + value: 0.082 + - type: mrr_at_1 + value: 29.140100000000004 + - type: mrr_at_3 + value: 33.703 + - type: mrr_at_5 + value: 34.7179 + - type: mrr_at_10 + value: 35.4443 + - type: mrr_at_20 + value: 35.830600000000004 + - type: mrr_at_100 + value: 36.1332 + - type: mrr_at_1000 + value: 36.1935 + - type: nauc_ndcg_at_1_max + value: 46.9222 + - type: nauc_ndcg_at_1_std + value: 3.3564999999999996 + - type: nauc_ndcg_at_1_diff1 + value: 60.583 + - type: nauc_ndcg_at_3_max + value: 49.205799999999996 + - type: nauc_ndcg_at_3_std + value: 5.976299999999999 + - type: nauc_ndcg_at_3_diff1 + value: 55.09610000000001 + - type: nauc_ndcg_at_5_max + value: 49.0533 + - type: nauc_ndcg_at_5_std + value: 6.5834 + - type: nauc_ndcg_at_5_diff1 + value: 54.430800000000005 + - type: nauc_ndcg_at_10_max + value: 48.626799999999996 + - type: nauc_ndcg_at_10_std + value: 7.4441 + - type: nauc_ndcg_at_10_diff1 + value: 53.1986 + - type: nauc_ndcg_at_20_max + value: 48.7498 + - type: nauc_ndcg_at_20_std + value: 8.3344 + - type: nauc_ndcg_at_20_diff1 + value: 52.844 + - type: nauc_ndcg_at_100_max + value: 48.7164 + - type: nauc_ndcg_at_100_std + value: 9.1646 + - type: nauc_ndcg_at_100_diff1 + value: 52.6307 + - type: nauc_ndcg_at_1000_max + value: 48.634699999999995 + - type: nauc_ndcg_at_1000_std + value: 9.3865 + - type: nauc_ndcg_at_1000_diff1 + value: 53.100899999999996 + - type: nauc_map_at_1_max + value: 46.9222 + - type: nauc_map_at_1_std + value: 3.3564999999999996 + - type: nauc_map_at_1_diff1 + value: 60.583 + - type: nauc_map_at_3_max + value: 48.7099 + - type: nauc_map_at_3_std + value: 5.2638 + - type: nauc_map_at_3_diff1 + value: 56.370200000000004 + - type: nauc_map_at_5_max + value: 48.6303 + - type: nauc_map_at_5_std + value: 5.5931 + - type: nauc_map_at_5_diff1 + value: 55.9968 + - type: nauc_map_at_10_max + value: 48.4549 + - type: nauc_map_at_10_std + value: 5.949800000000001 + - type: nauc_map_at_10_diff1 + value: 55.4941 + - type: nauc_map_at_20_max + value: 48.4854 + - type: nauc_map_at_20_std + value: 6.1861 + - type: nauc_map_at_20_diff1 + value: 55.4072 + - type: nauc_map_at_100_max + value: 48.4835 + - type: nauc_map_at_100_std + value: 6.2885 + - type: nauc_map_at_100_diff1 + value: 55.3743 + - type: nauc_map_at_1000_max + value: 48.4769 + - type: nauc_map_at_1000_std + value: 6.2978000000000005 + - type: nauc_map_at_1000_diff1 + value: 55.3852 + - type: nauc_recall_at_1_max + value: 46.9222 + - type: nauc_recall_at_1_std + value: 3.3564999999999996 + - type: nauc_recall_at_1_diff1 + value: 60.583 + - type: nauc_recall_at_3_max + value: 50.5754 + - type: nauc_recall_at_3_std + value: 8.005700000000001 + - type: nauc_recall_at_3_diff1 + value: 51.542100000000005 + - type: nauc_recall_at_5_max + value: 50.199000000000005 + - type: nauc_recall_at_5_std + value: 9.5088 + - type: nauc_recall_at_5_diff1 + value: 49.9358 + - type: nauc_recall_at_10_max + value: 48.899100000000004 + - type: nauc_recall_at_10_std + value: 12.2017 + - type: nauc_recall_at_10_diff1 + value: 46.042 + - type: nauc_recall_at_20_max + value: 49.433899999999994 + - type: nauc_recall_at_20_std + value: 16.1228 + - type: nauc_recall_at_20_diff1 + value: 44.1762 + - type: nauc_recall_at_100_max + value: 49.2626 + - type: nauc_recall_at_100_std + value: 23.1356 + - type: nauc_recall_at_100_diff1 + value: 41.2386 + - type: nauc_recall_at_1000_max + value: 48.7068 + - type: nauc_recall_at_1000_std + value: 34.4874 + - type: nauc_recall_at_1000_diff1 + value: 42.088 + - type: nauc_precision_at_1_max + value: 46.9222 + - type: nauc_precision_at_1_std + value: 3.3564999999999996 + - type: nauc_precision_at_1_diff1 + value: 60.583 + - type: nauc_precision_at_3_max + value: 50.5754 + - type: nauc_precision_at_3_std + value: 8.005700000000001 + - type: nauc_precision_at_3_diff1 + value: 51.542100000000005 + - type: nauc_precision_at_5_max + value: 50.199000000000005 + - type: nauc_precision_at_5_std + value: 9.5088 + - type: nauc_precision_at_5_diff1 + value: 49.9358 + - type: nauc_precision_at_10_max + value: 48.899100000000004 + - type: nauc_precision_at_10_std + value: 12.2017 + - type: nauc_precision_at_10_diff1 + value: 46.042 + - type: nauc_precision_at_20_max + value: 49.433899999999994 + - type: nauc_precision_at_20_std + value: 16.1228 + - type: nauc_precision_at_20_diff1 + value: 44.1762 + - type: nauc_precision_at_100_max + value: 49.2626 + - type: nauc_precision_at_100_std + value: 23.1356 + - type: nauc_precision_at_100_diff1 + value: 41.2386 + - type: nauc_precision_at_1000_max + value: 48.7068 + - type: nauc_precision_at_1000_std + value: 34.4874 + - type: nauc_precision_at_1000_diff1 + value: 42.088 + - type: nauc_mrr_at_1_max + value: 46.9222 + - type: nauc_mrr_at_1_std + value: 3.3564999999999996 + - type: nauc_mrr_at_1_diff1 + value: 60.583 + - type: nauc_mrr_at_3_max + value: 48.7099 + - type: nauc_mrr_at_3_std + value: 5.2638 + - type: nauc_mrr_at_3_diff1 + value: 56.370200000000004 + - type: nauc_mrr_at_5_max + value: 48.6303 + - type: nauc_mrr_at_5_std + value: 5.5931 + - type: nauc_mrr_at_5_diff1 + value: 55.9968 + - type: nauc_mrr_at_10_max + value: 48.4549 + - type: nauc_mrr_at_10_std + value: 5.949800000000001 + - type: nauc_mrr_at_10_diff1 + value: 55.4941 + - type: nauc_mrr_at_20_max + value: 48.4854 + - type: nauc_mrr_at_20_std + value: 6.1861 + - type: nauc_mrr_at_20_diff1 + value: 55.4072 + - type: nauc_mrr_at_100_max + value: 48.4835 + - type: nauc_mrr_at_100_std + value: 6.2885 + - type: nauc_mrr_at_100_diff1 + value: 55.3743 + - type: nauc_mrr_at_1000_max + value: 48.4769 + - type: nauc_mrr_at_1000_std + value: 6.2978000000000005 + - type: nauc_mrr_at_1000_diff1 + value: 55.3852 + - type: main_score + value: 38.778 task: type: Retrieval + - dataset: + config: go + name: MTEB COIRCodeSearchNetRetrieval (go) + revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 + split: test + type: CoIR-Retrieval/CodeSearchNet metrics: - type: ndcg_at_1 - value: 0.5571 + value: 42.809999999999995 + - type: ndcg_at_3 + value: 51.949999999999996 + - type: ndcg_at_5 + value: 54.217000000000006 - type: ndcg_at_10 - value: 0.49688 + value: 56.296 + - type: ndcg_at_20 + value: 57.735 - type: ndcg_at_100 - value: 0.60493 + value: 59.68599999999999 - type: ndcg_at_1000 - value: 0.62922 - - type: ndcg_at_20 - value: 0.54438 - - type: ndcg_at_3 - value: 0.47981 - - type: ndcg_at_5 - value: 0.46584 + value: 60.812 + - type: map_at_1 + value: 42.809999999999995 + - type: map_at_3 + value: 49.727 + - type: map_at_5 + value: 50.988 + - type: map_at_10 + value: 51.847 + - type: map_at_20 + value: 52.248000000000005 + - type: map_at_100 + value: 52.52 + - type: map_at_1000 + value: 52.561 - type: recall_at_1 - value: 0.1638 + value: 42.809999999999995 + - type: recall_at_3 + value: 58.372 + - type: recall_at_5 + value: 63.864 - type: recall_at_10 - value: 0.54155 + value: 70.291 + - type: recall_at_20 + value: 75.92999999999999 - type: recall_at_100 - value: 0.85136 + value: 86.432 - type: recall_at_1000 - value: 0.96951 - - type: recall_at_20 - value: 0.65329 - - type: recall_at_3 - value: 0.31503 - - type: recall_at_5 - value: 0.40356 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (fa) - config: fa - split: dev + value: 95.371 + - type: precision_at_1 + value: 42.809999999999995 + - type: precision_at_3 + value: 19.457 + - type: precision_at_5 + value: 12.773000000000001 + - type: precision_at_10 + value: 7.029000000000001 + - type: precision_at_20 + value: 3.7960000000000003 + - type: precision_at_100 + value: 0.864 + - type: precision_at_1000 + value: 0.095 + - type: mrr_at_1 + value: 42.8097 + - type: mrr_at_3 + value: 49.7271 + - type: mrr_at_5 + value: 50.987899999999996 + - type: mrr_at_10 + value: 51.847100000000005 + - type: mrr_at_20 + value: 52.2483 + - type: mrr_at_100 + value: 52.519499999999994 + - type: mrr_at_1000 + value: 52.560700000000004 + - type: nauc_ndcg_at_1_max + value: 42.5169 + - type: nauc_ndcg_at_1_std + value: -2.56 + - type: nauc_ndcg_at_1_diff1 + value: 61.5235 + - type: nauc_ndcg_at_3_max + value: 43.897999999999996 + - type: nauc_ndcg_at_3_std + value: -0.927 + - type: nauc_ndcg_at_3_diff1 + value: 55.5453 + - type: nauc_ndcg_at_5_max + value: 44.069199999999995 + - type: nauc_ndcg_at_5_std + value: -0.5125000000000001 + - type: nauc_ndcg_at_5_diff1 + value: 55.095000000000006 + - type: nauc_ndcg_at_10_max + value: 43.9261 + - type: nauc_ndcg_at_10_std + value: 0.218 + - type: nauc_ndcg_at_10_diff1 + value: 54.7159 + - type: nauc_ndcg_at_20_max + value: 44.0206 + - type: nauc_ndcg_at_20_std + value: 0.8718999999999999 + - type: nauc_ndcg_at_20_diff1 + value: 54.830400000000004 + - type: nauc_ndcg_at_100_max + value: 43.7526 + - type: nauc_ndcg_at_100_std + value: 0.9793 + - type: nauc_ndcg_at_100_diff1 + value: 54.9701 + - type: nauc_ndcg_at_1000_max + value: 43.8809 + - type: nauc_ndcg_at_1000_std + value: 0.7155 + - type: nauc_ndcg_at_1000_diff1 + value: 55.3053 + - type: nauc_map_at_1_max + value: 42.5169 + - type: nauc_map_at_1_std + value: -2.56 + - type: nauc_map_at_1_diff1 + value: 61.5235 + - type: nauc_map_at_3_max + value: 43.5908 + - type: nauc_map_at_3_std + value: -1.3469 + - type: nauc_map_at_3_diff1 + value: 56.9825 + - type: nauc_map_at_5_max + value: 43.674099999999996 + - type: nauc_map_at_5_std + value: -1.1391 + - type: nauc_map_at_5_diff1 + value: 56.7628 + - type: nauc_map_at_10_max + value: 43.6154 + - type: nauc_map_at_10_std + value: -0.861 + - type: nauc_map_at_10_diff1 + value: 56.6439 + - type: nauc_map_at_20_max + value: 43.650099999999995 + - type: nauc_map_at_20_std + value: -0.6788 + - type: nauc_map_at_20_diff1 + value: 56.6917 + - type: nauc_map_at_100_max + value: 43.6075 + - type: nauc_map_at_100_std + value: -0.6773 + - type: nauc_map_at_100_diff1 + value: 56.7132 + - type: nauc_map_at_1000_max + value: 43.6113 + - type: nauc_map_at_1000_std + value: -0.6847 + - type: nauc_map_at_1000_diff1 + value: 56.725300000000004 + - type: nauc_recall_at_1_max + value: 42.5169 + - type: nauc_recall_at_1_std + value: -2.56 + - type: nauc_recall_at_1_diff1 + value: 61.5235 + - type: nauc_recall_at_3_max + value: 44.8282 + - type: nauc_recall_at_3_std + value: 0.3731 + - type: nauc_recall_at_3_diff1 + value: 51.139199999999995 + - type: nauc_recall_at_5_max + value: 45.3912 + - type: nauc_recall_at_5_std + value: 1.6466999999999998 + - type: nauc_recall_at_5_diff1 + value: 49.5336 + - type: nauc_recall_at_10_max + value: 45.0172 + - type: nauc_recall_at_10_std + value: 4.702 + - type: nauc_recall_at_10_diff1 + value: 47.287600000000005 + - type: nauc_recall_at_20_max + value: 45.5956 + - type: nauc_recall_at_20_std + value: 8.8859 + - type: nauc_recall_at_20_diff1 + value: 46.5039 + - type: nauc_recall_at_100_max + value: 43.7193 + - type: nauc_recall_at_100_std + value: 15.4564 + - type: nauc_recall_at_100_diff1 + value: 42.9843 + - type: nauc_recall_at_1000_max + value: 49.6578 + - type: nauc_recall_at_1000_std + value: 28.1802 + - type: nauc_recall_at_1000_diff1 + value: 37.0098 + - type: nauc_precision_at_1_max + value: 42.5169 + - type: nauc_precision_at_1_std + value: -2.56 + - type: nauc_precision_at_1_diff1 + value: 61.5235 + - type: nauc_precision_at_3_max + value: 44.8282 + - type: nauc_precision_at_3_std + value: 0.3731 + - type: nauc_precision_at_3_diff1 + value: 51.139199999999995 + - type: nauc_precision_at_5_max + value: 45.3912 + - type: nauc_precision_at_5_std + value: 1.6466999999999998 + - type: nauc_precision_at_5_diff1 + value: 49.5336 + - type: nauc_precision_at_10_max + value: 45.0172 + - type: nauc_precision_at_10_std + value: 4.702 + - type: nauc_precision_at_10_diff1 + value: 47.287600000000005 + - type: nauc_precision_at_20_max + value: 45.5956 + - type: nauc_precision_at_20_std + value: 8.8859 + - type: nauc_precision_at_20_diff1 + value: 46.5039 + - type: nauc_precision_at_100_max + value: 43.7193 + - type: nauc_precision_at_100_std + value: 15.4564 + - type: nauc_precision_at_100_diff1 + value: 42.9843 + - type: nauc_precision_at_1000_max + value: 49.6578 + - type: nauc_precision_at_1000_std + value: 28.1802 + - type: nauc_precision_at_1000_diff1 + value: 37.0098 + - type: nauc_mrr_at_1_max + value: 42.5169 + - type: nauc_mrr_at_1_std + value: -2.56 + - type: nauc_mrr_at_1_diff1 + value: 61.5235 + - type: nauc_mrr_at_3_max + value: 43.5908 + - type: nauc_mrr_at_3_std + value: -1.3469 + - type: nauc_mrr_at_3_diff1 + value: 56.9825 + - type: nauc_mrr_at_5_max + value: 43.674099999999996 + - type: nauc_mrr_at_5_std + value: -1.1391 + - type: nauc_mrr_at_5_diff1 + value: 56.7628 + - type: nauc_mrr_at_10_max + value: 43.6154 + - type: nauc_mrr_at_10_std + value: -0.861 + - type: nauc_mrr_at_10_diff1 + value: 56.6439 + - type: nauc_mrr_at_20_max + value: 43.650099999999995 + - type: nauc_mrr_at_20_std + value: -0.6788 + - type: nauc_mrr_at_20_diff1 + value: 56.6917 + - type: nauc_mrr_at_100_max + value: 43.6075 + - type: nauc_mrr_at_100_std + value: -0.6773 + - type: nauc_mrr_at_100_diff1 + value: 56.7132 + - type: nauc_mrr_at_1000_max + value: 43.6113 + - type: nauc_mrr_at_1000_std + value: -0.6847 + - type: nauc_mrr_at_1000_diff1 + value: 56.725300000000004 + - type: main_score + value: 56.296 task: type: Retrieval + - dataset: + config: ruby + name: MTEB COIRCodeSearchNetRetrieval (ruby) + revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 + split: test + type: CoIR-Retrieval/CodeSearchNet metrics: - type: ndcg_at_1 - value: 0.39873 + value: 31.721 + - type: ndcg_at_3 + value: 38.559 + - type: ndcg_at_5 + value: 40.303 - type: ndcg_at_10 - value: 0.50226 + value: 42.536 + - type: ndcg_at_20 + value: 44.05 - type: ndcg_at_100 - value: 0.56517 + value: 46.565 - type: ndcg_at_1000 - value: 0.57967 - - type: ndcg_at_20 - value: 0.5292 - - type: ndcg_at_3 - value: 0.42738 - - type: ndcg_at_5 - value: 0.45843 + value: 48.447 + - type: map_at_1 + value: 31.721 + - type: map_at_3 + value: 36.915 + - type: map_at_5 + value: 37.891000000000005 + - type: map_at_10 + value: 38.814 + - type: map_at_20 + value: 39.236 + - type: map_at_100 + value: 39.574 + - type: map_at_1000 + value: 39.641999999999996 - type: recall_at_1 - value: 0.25369 + value: 31.721 + - type: recall_at_3 + value: 43.299 + - type: recall_at_5 + value: 47.502 - type: recall_at_10 - value: 0.63776 + value: 54.400999999999996 + - type: recall_at_20 + value: 60.349 - type: recall_at_100 - value: 0.87686 + value: 74.068 - type: recall_at_1000 - value: 0.9671 - - type: recall_at_20 - value: 0.72099 - - type: recall_at_3 - value: 0.43808 - - type: recall_at_5 - value: 0.52378 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (fi) - config: fi - split: dev + value: 89.056 + - type: precision_at_1 + value: 31.721 + - type: precision_at_3 + value: 14.433000000000002 + - type: precision_at_5 + value: 9.5 + - type: precision_at_10 + value: 5.4399999999999995 + - type: precision_at_20 + value: 3.017 + - type: precision_at_100 + value: 0.741 + - type: precision_at_1000 + value: 0.089 + - type: mrr_at_1 + value: 31.7209 + - type: mrr_at_3 + value: 36.9151 + - type: mrr_at_5 + value: 37.8906 + - type: mrr_at_10 + value: 38.8144 + - type: mrr_at_20 + value: 39.2355 + - type: mrr_at_100 + value: 39.5737 + - type: mrr_at_1000 + value: 39.641999999999996 + - type: nauc_ndcg_at_1_max + value: 46.428999999999995 + - type: nauc_ndcg_at_1_std + value: 0.0014 + - type: nauc_ndcg_at_1_diff1 + value: 59.6017 + - type: nauc_ndcg_at_3_max + value: 45.9805 + - type: nauc_ndcg_at_3_std + value: 0.5511 + - type: nauc_ndcg_at_3_diff1 + value: 53.4978 + - type: nauc_ndcg_at_5_max + value: 45.5339 + - type: nauc_ndcg_at_5_std + value: 1.2229 + - type: nauc_ndcg_at_5_diff1 + value: 51.798500000000004 + - type: nauc_ndcg_at_10_max + value: 44.018 + - type: nauc_ndcg_at_10_std + value: 1.6709 + - type: nauc_ndcg_at_10_diff1 + value: 50.428799999999995 + - type: nauc_ndcg_at_20_max + value: 43.5252 + - type: nauc_ndcg_at_20_std + value: 2.4627 + - type: nauc_ndcg_at_20_diff1 + value: 50.6172 + - type: nauc_ndcg_at_100_max + value: 43.723099999999995 + - type: nauc_ndcg_at_100_std + value: 4.0416 + - type: nauc_ndcg_at_100_diff1 + value: 50.135600000000004 + - type: nauc_ndcg_at_1000_max + value: 43.7739 + - type: nauc_ndcg_at_1000_std + value: 3.4729 + - type: nauc_ndcg_at_1000_diff1 + value: 50.6595 + - type: nauc_map_at_1_max + value: 46.428999999999995 + - type: nauc_map_at_1_std + value: 0.0014 + - type: nauc_map_at_1_diff1 + value: 59.6017 + - type: nauc_map_at_3_max + value: 46.217999999999996 + - type: nauc_map_at_3_std + value: 0.43889999999999996 + - type: nauc_map_at_3_diff1 + value: 54.882299999999994 + - type: nauc_map_at_5_max + value: 45.9757 + - type: nauc_map_at_5_std + value: 0.8049999999999999 + - type: nauc_map_at_5_diff1 + value: 53.950900000000004 + - type: nauc_map_at_10_max + value: 45.3363 + - type: nauc_map_at_10_std + value: 0.9662999999999999 + - type: nauc_map_at_10_diff1 + value: 53.369 + - type: nauc_map_at_20_max + value: 45.2008 + - type: nauc_map_at_20_std + value: 1.1801000000000001 + - type: nauc_map_at_20_diff1 + value: 53.4425 + - type: nauc_map_at_100_max + value: 45.226699999999994 + - type: nauc_map_at_100_std + value: 1.3667 + - type: nauc_map_at_100_diff1 + value: 53.4089 + - type: nauc_map_at_1000_max + value: 45.2252 + - type: nauc_map_at_1000_std + value: 1.3433000000000002 + - type: nauc_map_at_1000_diff1 + value: 53.4268 + - type: nauc_recall_at_1_max + value: 46.428999999999995 + - type: nauc_recall_at_1_std + value: 0.0014 + - type: nauc_recall_at_1_diff1 + value: 59.6017 + - type: nauc_recall_at_3_max + value: 45.2499 + - type: nauc_recall_at_3_std + value: 0.8637 + - type: nauc_recall_at_3_diff1 + value: 49.5773 + - type: nauc_recall_at_5_max + value: 44.1355 + - type: nauc_recall_at_5_std + value: 2.5255 + - type: nauc_recall_at_5_diff1 + value: 45.3656 + - type: nauc_recall_at_10_max + value: 39.313700000000004 + - type: nauc_recall_at_10_std + value: 4.1421 + - type: nauc_recall_at_10_diff1 + value: 40.8109 + - type: nauc_recall_at_20_max + value: 36.923 + - type: nauc_recall_at_20_std + value: 7.691199999999999 + - type: nauc_recall_at_20_diff1 + value: 40.8715 + - type: nauc_recall_at_100_max + value: 36.296 + - type: nauc_recall_at_100_std + value: 22.020999999999997 + - type: nauc_recall_at_100_diff1 + value: 33.400800000000004 + - type: nauc_recall_at_1000_max + value: 30.508999999999997 + - type: nauc_recall_at_1000_std + value: 29.497600000000002 + - type: nauc_recall_at_1000_diff1 + value: 27.5001 + - type: nauc_precision_at_1_max + value: 46.428999999999995 + - type: nauc_precision_at_1_std + value: 0.0014 + - type: nauc_precision_at_1_diff1 + value: 59.6017 + - type: nauc_precision_at_3_max + value: 45.2499 + - type: nauc_precision_at_3_std + value: 0.8637 + - type: nauc_precision_at_3_diff1 + value: 49.5773 + - type: nauc_precision_at_5_max + value: 44.1355 + - type: nauc_precision_at_5_std + value: 2.5255 + - type: nauc_precision_at_5_diff1 + value: 45.3656 + - type: nauc_precision_at_10_max + value: 39.313700000000004 + - type: nauc_precision_at_10_std + value: 4.1421 + - type: nauc_precision_at_10_diff1 + value: 40.8109 + - type: nauc_precision_at_20_max + value: 36.923 + - type: nauc_precision_at_20_std + value: 7.691199999999999 + - type: nauc_precision_at_20_diff1 + value: 40.8715 + - type: nauc_precision_at_100_max + value: 36.296 + - type: nauc_precision_at_100_std + value: 22.020999999999997 + - type: nauc_precision_at_100_diff1 + value: 33.400800000000004 + - type: nauc_precision_at_1000_max + value: 30.508999999999997 + - type: nauc_precision_at_1000_std + value: 29.497600000000002 + - type: nauc_precision_at_1000_diff1 + value: 27.5001 + - type: nauc_mrr_at_1_max + value: 46.428999999999995 + - type: nauc_mrr_at_1_std + value: 0.0014 + - type: nauc_mrr_at_1_diff1 + value: 59.6017 + - type: nauc_mrr_at_3_max + value: 46.217999999999996 + - type: nauc_mrr_at_3_std + value: 0.43889999999999996 + - type: nauc_mrr_at_3_diff1 + value: 54.882299999999994 + - type: nauc_mrr_at_5_max + value: 45.9757 + - type: nauc_mrr_at_5_std + value: 0.8049999999999999 + - type: nauc_mrr_at_5_diff1 + value: 53.950900000000004 + - type: nauc_mrr_at_10_max + value: 45.3363 + - type: nauc_mrr_at_10_std + value: 0.9662999999999999 + - type: nauc_mrr_at_10_diff1 + value: 53.369 + - type: nauc_mrr_at_20_max + value: 45.2008 + - type: nauc_mrr_at_20_std + value: 1.1801000000000001 + - type: nauc_mrr_at_20_diff1 + value: 53.4425 + - type: nauc_mrr_at_100_max + value: 45.226699999999994 + - type: nauc_mrr_at_100_std + value: 1.3667 + - type: nauc_mrr_at_100_diff1 + value: 53.4089 + - type: nauc_mrr_at_1000_max + value: 45.2252 + - type: nauc_mrr_at_1000_std + value: 1.3433000000000002 + - type: nauc_mrr_at_1000_diff1 + value: 53.4268 + - type: main_score + value: 42.536 task: type: Retrieval + - dataset: + config: java + name: MTEB COIRCodeSearchNetRetrieval (java) + revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 + split: test + type: CoIR-Retrieval/CodeSearchNet metrics: - type: ndcg_at_1 - value: 0.60818 + value: 36.887 + - type: ndcg_at_3 + value: 44.671 + - type: ndcg_at_5 + value: 46.619 - type: ndcg_at_10 - value: 0.6746 + value: 48.54 + - type: ndcg_at_20 + value: 49.881 - type: ndcg_at_100 - value: 0.71516 + value: 51.847 - type: ndcg_at_1000 - value: 0.7218 - - type: ndcg_at_20 - value: 0.69692 - - type: ndcg_at_3 - value: 0.6006 - - type: ndcg_at_5 - value: 0.63842 + value: 53.286 + - type: map_at_1 + value: 36.887 + - type: map_at_3 + value: 42.805 + - type: map_at_5 + value: 43.884 + - type: map_at_10 + value: 44.68 + - type: map_at_20 + value: 45.051 + - type: map_at_100 + value: 45.316 + - type: map_at_1000 + value: 45.364 - type: recall_at_1 - value: 0.39264 + value: 36.887 + - type: recall_at_3 + value: 50.05 + - type: recall_at_5 + value: 54.788000000000004 - type: recall_at_10 - value: 0.78577 + value: 60.711999999999996 + - type: recall_at_20 + value: 65.997 - type: recall_at_100 - value: 0.93291 + value: 76.696 - type: recall_at_1000 - value: 0.97493 - - type: recall_at_20 - value: 0.85435 - - type: recall_at_3 - value: 0.61055 - - type: recall_at_5 - value: 0.69774 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (fr) - config: fr - split: dev + value: 88.371 + - type: precision_at_1 + value: 36.887 + - type: precision_at_3 + value: 16.683 + - type: precision_at_5 + value: 10.958 + - type: precision_at_10 + value: 6.071 + - type: precision_at_20 + value: 3.3000000000000003 + - type: precision_at_100 + value: 0.767 + - type: precision_at_1000 + value: 0.08800000000000001 + - type: mrr_at_1 + value: 36.9147 + - type: mrr_at_3 + value: 42.823699999999995 + - type: mrr_at_5 + value: 43.8985 + - type: mrr_at_10 + value: 44.6961 + - type: mrr_at_20 + value: 45.067 + - type: mrr_at_100 + value: 45.3318 + - type: mrr_at_1000 + value: 45.3801 + - type: nauc_ndcg_at_1_max + value: 42.8063 + - type: nauc_ndcg_at_1_std + value: -5.3001 + - type: nauc_ndcg_at_1_diff1 + value: 63.370099999999994 + - type: nauc_ndcg_at_3_max + value: 44.0649 + - type: nauc_ndcg_at_3_std + value: -4.0304 + - type: nauc_ndcg_at_3_diff1 + value: 57.7429 + - type: nauc_ndcg_at_5_max + value: 43.864799999999995 + - type: nauc_ndcg_at_5_std + value: -3.2800000000000002 + - type: nauc_ndcg_at_5_diff1 + value: 57.0472 + - type: nauc_ndcg_at_10_max + value: 43.614799999999995 + - type: nauc_ndcg_at_10_std + value: -2.424 + - type: nauc_ndcg_at_10_diff1 + value: 56.3498 + - type: nauc_ndcg_at_20_max + value: 43.6108 + - type: nauc_ndcg_at_20_std + value: -1.699 + - type: nauc_ndcg_at_20_diff1 + value: 56.2153 + - type: nauc_ndcg_at_100_max + value: 43.4705 + - type: nauc_ndcg_at_100_std + value: -0.7144 + - type: nauc_ndcg_at_100_diff1 + value: 56.0679 + - type: nauc_ndcg_at_1000_max + value: 43.6856 + - type: nauc_ndcg_at_1000_std + value: -0.7129 + - type: nauc_ndcg_at_1000_diff1 + value: 56.40540000000001 + - type: nauc_map_at_1_max + value: 42.8063 + - type: nauc_map_at_1_std + value: -5.3001 + - type: nauc_map_at_1_diff1 + value: 63.370099999999994 + - type: nauc_map_at_3_max + value: 43.797999999999995 + - type: nauc_map_at_3_std + value: -4.3491 + - type: nauc_map_at_3_diff1 + value: 59.0673 + - type: nauc_map_at_5_max + value: 43.6812 + - type: nauc_map_at_5_std + value: -3.9397 + - type: nauc_map_at_5_diff1 + value: 58.6982 + - type: nauc_map_at_10_max + value: 43.5745 + - type: nauc_map_at_10_std + value: -3.6122 + - type: nauc_map_at_10_diff1 + value: 58.431999999999995 + - type: nauc_map_at_20_max + value: 43.573 + - type: nauc_map_at_20_std + value: -3.4323 + - type: nauc_map_at_20_diff1 + value: 58.4168 + - type: nauc_map_at_100_max + value: 43.5448 + - type: nauc_map_at_100_std + value: -3.3167 + - type: nauc_map_at_100_diff1 + value: 58.394999999999996 + - type: nauc_map_at_1000_max + value: 43.5506 + - type: nauc_map_at_1000_std + value: -3.3144 + - type: nauc_map_at_1000_diff1 + value: 58.4057 + - type: nauc_recall_at_1_max + value: 42.8063 + - type: nauc_recall_at_1_std + value: -5.3001 + - type: nauc_recall_at_1_diff1 + value: 63.370099999999994 + - type: nauc_recall_at_3_max + value: 44.8286 + - type: nauc_recall_at_3_std + value: -3.0949999999999998 + - type: nauc_recall_at_3_diff1 + value: 53.8907 + - type: nauc_recall_at_5_max + value: 44.3801 + - type: nauc_recall_at_5_std + value: -1.1593 + - type: nauc_recall_at_5_diff1 + value: 51.948899999999995 + - type: nauc_recall_at_10_max + value: 43.6005 + - type: nauc_recall_at_10_std + value: 1.9532999999999998 + - type: nauc_recall_at_10_diff1 + value: 49.2211 + - type: nauc_recall_at_20_max + value: 43.5839 + - type: nauc_recall_at_20_std + value: 5.8288 + - type: nauc_recall_at_20_diff1 + value: 47.7761 + - type: nauc_recall_at_100_max + value: 42.6633 + - type: nauc_recall_at_100_std + value: 16.4317 + - type: nauc_recall_at_100_diff1 + value: 44.0676 + - type: nauc_recall_at_1000_max + value: 46.698 + - type: nauc_recall_at_1000_std + value: 30.054799999999997 + - type: nauc_recall_at_1000_diff1 + value: 41.5816 + - type: nauc_precision_at_1_max + value: 42.8063 + - type: nauc_precision_at_1_std + value: -5.3001 + - type: nauc_precision_at_1_diff1 + value: 63.370099999999994 + - type: nauc_precision_at_3_max + value: 44.8286 + - type: nauc_precision_at_3_std + value: -3.0949999999999998 + - type: nauc_precision_at_3_diff1 + value: 53.8907 + - type: nauc_precision_at_5_max + value: 44.3801 + - type: nauc_precision_at_5_std + value: -1.1593 + - type: nauc_precision_at_5_diff1 + value: 51.948899999999995 + - type: nauc_precision_at_10_max + value: 43.6005 + - type: nauc_precision_at_10_std + value: 1.9532999999999998 + - type: nauc_precision_at_10_diff1 + value: 49.2211 + - type: nauc_precision_at_20_max + value: 43.5839 + - type: nauc_precision_at_20_std + value: 5.8288 + - type: nauc_precision_at_20_diff1 + value: 47.7761 + - type: nauc_precision_at_100_max + value: 42.6633 + - type: nauc_precision_at_100_std + value: 16.4317 + - type: nauc_precision_at_100_diff1 + value: 44.0676 + - type: nauc_precision_at_1000_max + value: 46.698 + - type: nauc_precision_at_1000_std + value: 30.054799999999997 + - type: nauc_precision_at_1000_diff1 + value: 41.5816 + - type: nauc_mrr_at_1_max + value: 42.7425 + - type: nauc_mrr_at_1_std + value: -5.2358 + - type: nauc_mrr_at_1_diff1 + value: 63.285199999999996 + - type: nauc_mrr_at_3_max + value: 43.763200000000005 + - type: nauc_mrr_at_3_std + value: -4.2973 + - type: nauc_mrr_at_3_diff1 + value: 59.031 + - type: nauc_mrr_at_5_max + value: 43.650800000000004 + - type: nauc_mrr_at_5_std + value: -3.8918 + - type: nauc_mrr_at_5_diff1 + value: 58.6636 + - type: nauc_mrr_at_10_max + value: 43.5429 + - type: nauc_mrr_at_10_std + value: -3.5659000000000005 + - type: nauc_mrr_at_10_diff1 + value: 58.3946 + - type: nauc_mrr_at_20_max + value: 43.5411 + - type: nauc_mrr_at_20_std + value: -3.3855000000000004 + - type: nauc_mrr_at_20_diff1 + value: 58.379099999999994 + - type: nauc_mrr_at_100_max + value: 43.5128 + - type: nauc_mrr_at_100_std + value: -3.2696000000000005 + - type: nauc_mrr_at_100_diff1 + value: 58.3572 + - type: nauc_mrr_at_1000_max + value: 43.5186 + - type: nauc_mrr_at_1000_std + value: -3.2672 + - type: nauc_mrr_at_1000_diff1 + value: 58.3678 + - type: main_score + value: 48.54 task: type: Retrieval + - dataset: + config: php + name: MTEB COIRCodeSearchNetRetrieval (php) + revision: 4adc7bc41202b5c13543c9c886a25f340634dab3 + split: test + type: CoIR-Retrieval/CodeSearchNet metrics: - type: ndcg_at_1 - value: 0.3965 + value: 30.734 + - type: ndcg_at_3 + value: 38.155 + - type: ndcg_at_5 + value: 40.306999999999995 - type: ndcg_at_10 - value: 0.49891 + value: 42.510999999999996 + - type: ndcg_at_20 + value: 44.156 - type: ndcg_at_100 - value: 0.56492 + value: 46.641 - type: ndcg_at_1000 - value: 0.57837 - - type: ndcg_at_20 - value: 0.53163 - - type: ndcg_at_3 - value: 0.39843 - - type: ndcg_at_5 - value: 0.44416 + value: 48.359 + - type: map_at_1 + value: 30.734 + - type: map_at_3 + value: 36.347 + - type: map_at_5 + value: 37.539 + - type: map_at_10 + value: 38.455 + - type: map_at_20 + value: 38.906 + - type: map_at_100 + value: 39.24 + - type: map_at_1000 + value: 39.300000000000004 - type: recall_at_1 - value: 0.22644 + value: 30.734 + - type: recall_at_3 + value: 43.378 + - type: recall_at_5 + value: 48.616 - type: recall_at_10 - value: 0.65169 + value: 55.395 + - type: recall_at_20 + value: 61.91 - type: recall_at_100 - value: 0.89786 + value: 75.432 - type: recall_at_1000 - value: 0.98081 - - type: recall_at_20 - value: 0.75338 - - type: recall_at_3 - value: 0.39798 - - type: recall_at_5 - value: 0.51001 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (hi) - config: hi - split: dev + value: 89.254 + - type: precision_at_1 + value: 30.734 + - type: precision_at_3 + value: 14.459 + - type: precision_at_5 + value: 9.722999999999999 + - type: precision_at_10 + value: 5.539000000000001 + - type: precision_at_20 + value: 3.0949999999999998 + - type: precision_at_100 + value: 0.754 + - type: precision_at_1000 + value: 0.089 + - type: mrr_at_1 + value: 30.6907 + - type: mrr_at_3 + value: 36.3137 + - type: mrr_at_5 + value: 37.5121 + - type: mrr_at_10 + value: 38.4289 + - type: mrr_at_20 + value: 38.8786 + - type: mrr_at_100 + value: 39.2136 + - type: mrr_at_1000 + value: 39.2729 + - type: nauc_ndcg_at_1_max + value: 36.8055 + - type: nauc_ndcg_at_1_std + value: -1.5909 + - type: nauc_ndcg_at_1_diff1 + value: 55.9244 + - type: nauc_ndcg_at_3_max + value: 38.4262 + - type: nauc_ndcg_at_3_std + value: 0.5292 + - type: nauc_ndcg_at_3_diff1 + value: 49.7477 + - type: nauc_ndcg_at_5_max + value: 38.0552 + - type: nauc_ndcg_at_5_std + value: 1.102 + - type: nauc_ndcg_at_5_diff1 + value: 48.5308 + - type: nauc_ndcg_at_10_max + value: 38.0054 + - type: nauc_ndcg_at_10_std + value: 1.9313 + - type: nauc_ndcg_at_10_diff1 + value: 48.016999999999996 + - type: nauc_ndcg_at_20_max + value: 37.8808 + - type: nauc_ndcg_at_20_std + value: 2.56 + - type: nauc_ndcg_at_20_diff1 + value: 47.5649 + - type: nauc_ndcg_at_100_max + value: 38.3754 + - type: nauc_ndcg_at_100_std + value: 3.6703 + - type: nauc_ndcg_at_100_diff1 + value: 47.6154 + - type: nauc_ndcg_at_1000_max + value: 38.534600000000005 + - type: nauc_ndcg_at_1000_std + value: 3.7317000000000005 + - type: nauc_ndcg_at_1000_diff1 + value: 48.0299 + - type: nauc_map_at_1_max + value: 36.8055 + - type: nauc_map_at_1_std + value: -1.5909 + - type: nauc_map_at_1_diff1 + value: 55.9244 + - type: nauc_map_at_3_max + value: 38.0383 + - type: nauc_map_at_3_std + value: 0.0207 + - type: nauc_map_at_3_diff1 + value: 51.137299999999996 + - type: nauc_map_at_5_max + value: 37.8223 + - type: nauc_map_at_5_std + value: 0.3179 + - type: nauc_map_at_5_diff1 + value: 50.4641 + - type: nauc_map_at_10_max + value: 37.8022 + - type: nauc_map_at_10_std + value: 0.6617999999999999 + - type: nauc_map_at_10_diff1 + value: 50.269 + - type: nauc_map_at_20_max + value: 37.7686 + - type: nauc_map_at_20_std + value: 0.8326999999999999 + - type: nauc_map_at_20_diff1 + value: 50.153499999999994 + - type: nauc_map_at_100_max + value: 37.832300000000004 + - type: nauc_map_at_100_std + value: 0.9767 + - type: nauc_map_at_100_diff1 + value: 50.174099999999996 + - type: nauc_map_at_1000_max + value: 37.838300000000004 + - type: nauc_map_at_1000_std + value: 0.9815 + - type: nauc_map_at_1000_diff1 + value: 50.1882 + - type: nauc_recall_at_1_max + value: 36.8055 + - type: nauc_recall_at_1_std + value: -1.5909 + - type: nauc_recall_at_1_diff1 + value: 55.9244 + - type: nauc_recall_at_3_max + value: 39.5304 + - type: nauc_recall_at_3_std + value: 1.9767 + - type: nauc_recall_at_3_diff1 + value: 45.8281 + - type: nauc_recall_at_5_max + value: 38.6851 + - type: nauc_recall_at_5_std + value: 3.4711 + - type: nauc_recall_at_5_diff1 + value: 42.8172 + - type: nauc_recall_at_10_max + value: 38.5524 + - type: nauc_recall_at_10_std + value: 6.2315000000000005 + - type: nauc_recall_at_10_diff1 + value: 40.801 + - type: nauc_recall_at_20_max + value: 38.048300000000005 + - type: nauc_recall_at_20_std + value: 9.3045 + - type: nauc_recall_at_20_diff1 + value: 38.222 + - type: nauc_recall_at_100_max + value: 42.054399999999994 + - type: nauc_recall_at_100_std + value: 20.4425 + - type: nauc_recall_at_100_diff1 + value: 35.0773 + - type: nauc_recall_at_1000_max + value: 49.2856 + - type: nauc_recall_at_1000_std + value: 38.4529 + - type: nauc_recall_at_1000_diff1 + value: 31.7647 + - type: nauc_precision_at_1_max + value: 36.8055 + - type: nauc_precision_at_1_std + value: -1.5909 + - type: nauc_precision_at_1_diff1 + value: 55.9244 + - type: nauc_precision_at_3_max + value: 39.5304 + - type: nauc_precision_at_3_std + value: 1.9767 + - type: nauc_precision_at_3_diff1 + value: 45.8281 + - type: nauc_precision_at_5_max + value: 38.6851 + - type: nauc_precision_at_5_std + value: 3.4711 + - type: nauc_precision_at_5_diff1 + value: 42.8172 + - type: nauc_precision_at_10_max + value: 38.5524 + - type: nauc_precision_at_10_std + value: 6.2315000000000005 + - type: nauc_precision_at_10_diff1 + value: 40.801 + - type: nauc_precision_at_20_max + value: 38.048300000000005 + - type: nauc_precision_at_20_std + value: 9.3045 + - type: nauc_precision_at_20_diff1 + value: 38.222 + - type: nauc_precision_at_100_max + value: 42.054399999999994 + - type: nauc_precision_at_100_std + value: 20.4425 + - type: nauc_precision_at_100_diff1 + value: 35.0773 + - type: nauc_precision_at_1000_max + value: 49.2856 + - type: nauc_precision_at_1000_std + value: 38.4529 + - type: nauc_precision_at_1000_diff1 + value: 31.7647 + - type: nauc_mrr_at_1_max + value: 36.8365 + - type: nauc_mrr_at_1_std + value: -1.4754 + - type: nauc_mrr_at_1_diff1 + value: 56.0597 + - type: nauc_mrr_at_3_max + value: 38.054 + - type: nauc_mrr_at_3_std + value: 0.09430000000000001 + - type: nauc_mrr_at_3_diff1 + value: 51.2016 + - type: nauc_mrr_at_5_max + value: 37.8431 + - type: nauc_mrr_at_5_std + value: 0.3829 + - type: nauc_mrr_at_5_diff1 + value: 50.5285 + - type: nauc_mrr_at_10_max + value: 37.8231 + - type: nauc_mrr_at_10_std + value: 0.7271 + - type: nauc_mrr_at_10_diff1 + value: 50.333099999999995 + - type: nauc_mrr_at_20_max + value: 37.7905 + - type: nauc_mrr_at_20_std + value: 0.8992999999999999 + - type: nauc_mrr_at_20_diff1 + value: 50.2181 + - type: nauc_mrr_at_100_max + value: 37.853500000000004 + - type: nauc_mrr_at_100_std + value: 1.0428 + - type: nauc_mrr_at_100_diff1 + value: 50.239 + - type: nauc_mrr_at_1000_max + value: 37.859500000000004 + - type: nauc_mrr_at_1000_std + value: 1.0477 + - type: nauc_mrr_at_1000_diff1 + value: 50.2532 + - type: main_score + value: 42.510999999999996 task: type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackAndroidRetrieval (default) + revision: f46a197baaae43b4f621051089b82a364682dfeb + split: test + type: mteb/cqadupstack-android metrics: - type: ndcg_at_1 - value: 0.36857 + value: 42.918 + - type: ndcg_at_3 + value: 47.992000000000004 + - type: ndcg_at_5 + value: 50.298 - type: ndcg_at_10 - value: 0.46141 + value: 53.047999999999995 + - type: ndcg_at_20 + value: 55.36600000000001 - type: ndcg_at_100 - value: 0.52565 + value: 58.18 - type: ndcg_at_1000 - value: 0.54319 - - type: ndcg_at_20 - value: 0.49384 - - type: ndcg_at_3 - value: 0.39469 - - type: ndcg_at_5 - value: 0.4184 + value: 59.992999999999995 + - type: map_at_1 + value: 35.147 + - type: map_at_3 + value: 42.985 + - type: map_at_5 + value: 44.895 + - type: map_at_10 + value: 46.568 + - type: map_at_20 + value: 47.527 + - type: map_at_100 + value: 48.178 + - type: map_at_1000 + value: 48.303000000000004 - type: recall_at_1 - value: 0.20185 + value: 35.147 + - type: recall_at_3 + value: 50.229 + - type: recall_at_5 + value: 56.586999999999996 - type: recall_at_10 - value: 0.59474 + value: 64.656 + - type: recall_at_20 + value: 72.875 - type: recall_at_100 - value: 0.83385 + value: 85.397 - type: recall_at_1000 - value: 0.94813 - - type: recall_at_20 - value: 0.69437 - - type: recall_at_3 - value: 0.38993 - - type: recall_at_5 - value: 0.47881 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (id) - config: id - split: dev + value: 96.799 + - type: precision_at_1 + value: 42.918 + - type: precision_at_3 + value: 22.698999999999998 + - type: precision_at_5 + value: 16.309 + - type: precision_at_10 + value: 10.100000000000001 + - type: precision_at_20 + value: 6.0440000000000005 + - type: precision_at_100 + value: 1.5890000000000002 + - type: precision_at_1000 + value: 0.209 + - type: mrr_at_1 + value: 42.9185 + - type: mrr_at_3 + value: 50.1907 + - type: mrr_at_5 + value: 51.9003 + - type: mrr_at_10 + value: 52.824400000000004 + - type: mrr_at_20 + value: 53.3002 + - type: mrr_at_100 + value: 53.5134 + - type: mrr_at_1000 + value: 53.5569 + - type: nauc_ndcg_at_1_max + value: 45.115300000000005 + - type: nauc_ndcg_at_1_std + value: -5.3469999999999995 + - type: nauc_ndcg_at_1_diff1 + value: 50.792899999999996 + - type: nauc_ndcg_at_3_max + value: 44.379000000000005 + - type: nauc_ndcg_at_3_std + value: -2.628 + - type: nauc_ndcg_at_3_diff1 + value: 45.6678 + - type: nauc_ndcg_at_5_max + value: 44.8852 + - type: nauc_ndcg_at_5_std + value: -1.7051 + - type: nauc_ndcg_at_5_diff1 + value: 46.0814 + - type: nauc_ndcg_at_10_max + value: 43.969500000000004 + - type: nauc_ndcg_at_10_std + value: -0.4902 + - type: nauc_ndcg_at_10_diff1 + value: 46.2439 + - type: nauc_ndcg_at_20_max + value: 44.588499999999996 + - type: nauc_ndcg_at_20_std + value: 0.5193 + - type: nauc_ndcg_at_20_diff1 + value: 45.9229 + - type: nauc_ndcg_at_100_max + value: 45.0779 + - type: nauc_ndcg_at_100_std + value: 1.1967999999999999 + - type: nauc_ndcg_at_100_diff1 + value: 46.090199999999996 + - type: nauc_ndcg_at_1000_max + value: 45.082 + - type: nauc_ndcg_at_1000_std + value: 0.3457 + - type: nauc_ndcg_at_1000_diff1 + value: 46.366 + - type: nauc_map_at_1_max + value: 38.731 + - type: nauc_map_at_1_std + value: -7.1701 + - type: nauc_map_at_1_diff1 + value: 52.0087 + - type: nauc_map_at_3_max + value: 42.126799999999996 + - type: nauc_map_at_3_std + value: -4.8249 + - type: nauc_map_at_3_diff1 + value: 47.7841 + - type: nauc_map_at_5_max + value: 43.2155 + - type: nauc_map_at_5_std + value: -3.9702 + - type: nauc_map_at_5_diff1 + value: 47.9376 + - type: nauc_map_at_10_max + value: 43.4398 + - type: nauc_map_at_10_std + value: -2.8201 + - type: nauc_map_at_10_diff1 + value: 47.9726 + - type: nauc_map_at_20_max + value: 43.9625 + - type: nauc_map_at_20_std + value: -2.4088 + - type: nauc_map_at_20_diff1 + value: 47.7323 + - type: nauc_map_at_100_max + value: 44.0439 + - type: nauc_map_at_100_std + value: -2.1932 + - type: nauc_map_at_100_diff1 + value: 47.672399999999996 + - type: nauc_map_at_1000_max + value: 44.059599999999996 + - type: nauc_map_at_1000_std + value: -2.2453999999999996 + - type: nauc_map_at_1000_diff1 + value: 47.6659 + - type: nauc_recall_at_1_max + value: 38.731 + - type: nauc_recall_at_1_std + value: -7.1701 + - type: nauc_recall_at_1_diff1 + value: 52.0087 + - type: nauc_recall_at_3_max + value: 40.5229 + - type: nauc_recall_at_3_std + value: -1.3240999999999998 + - type: nauc_recall_at_3_diff1 + value: 41.1764 + - type: nauc_recall_at_5_max + value: 41.248000000000005 + - type: nauc_recall_at_5_std + value: 1.4647999999999999 + - type: nauc_recall_at_5_diff1 + value: 41.044799999999995 + - type: nauc_recall_at_10_max + value: 38.6375 + - type: nauc_recall_at_10_std + value: 5.3439 + - type: nauc_recall_at_10_diff1 + value: 39.8162 + - type: nauc_recall_at_20_max + value: 39.6813 + - type: nauc_recall_at_20_std + value: 11.1138 + - type: nauc_recall_at_20_diff1 + value: 36.8881 + - type: nauc_recall_at_100_max + value: 44.9346 + - type: nauc_recall_at_100_std + value: 22.5203 + - type: nauc_recall_at_100_diff1 + value: 34.8792 + - type: nauc_recall_at_1000_max + value: 52.49979999999999 + - type: nauc_recall_at_1000_std + value: 50.954299999999996 + - type: nauc_recall_at_1000_diff1 + value: 36.1016 + - type: nauc_precision_at_1_max + value: 45.115300000000005 + - type: nauc_precision_at_1_std + value: -5.3469999999999995 + - type: nauc_precision_at_1_diff1 + value: 50.792899999999996 + - type: nauc_precision_at_3_max + value: 41.841 + - type: nauc_precision_at_3_std + value: 3.3930000000000002 + - type: nauc_precision_at_3_diff1 + value: 27.495399999999997 + - type: nauc_precision_at_5_max + value: 38.527 + - type: nauc_precision_at_5_std + value: 8.2496 + - type: nauc_precision_at_5_diff1 + value: 19.3628 + - type: nauc_precision_at_10_max + value: 27.5499 + - type: nauc_precision_at_10_std + value: 13.264100000000001 + - type: nauc_precision_at_10_diff1 + value: 9.9718 + - type: nauc_precision_at_20_max + value: 21.431 + - type: nauc_precision_at_20_std + value: 14.426400000000001 + - type: nauc_precision_at_20_diff1 + value: -0.11030000000000001 + - type: nauc_precision_at_100_max + value: 6.8088 + - type: nauc_precision_at_100_std + value: 9.8979 + - type: nauc_precision_at_100_diff1 + value: -10.1603 + - type: nauc_precision_at_1000_max + value: -6.4949 + - type: nauc_precision_at_1000_std + value: -3.9967999999999995 + - type: nauc_precision_at_1000_diff1 + value: -17.765800000000002 + - type: nauc_mrr_at_1_max + value: 45.115300000000005 + - type: nauc_mrr_at_1_std + value: -5.3469999999999995 + - type: nauc_mrr_at_1_diff1 + value: 50.792899999999996 + - type: nauc_mrr_at_3_max + value: 45.8581 + - type: nauc_mrr_at_3_std + value: -2.9239 + - type: nauc_mrr_at_3_diff1 + value: 47.079 + - type: nauc_mrr_at_5_max + value: 45.5453 + - type: nauc_mrr_at_5_std + value: -2.2778 + - type: nauc_mrr_at_5_diff1 + value: 47.0394 + - type: nauc_mrr_at_10_max + value: 45.2727 + - type: nauc_mrr_at_10_std + value: -2.1793 + - type: nauc_mrr_at_10_diff1 + value: 46.7719 + - type: nauc_mrr_at_20_max + value: 45.232 + - type: nauc_mrr_at_20_std + value: -2.0842 + - type: nauc_mrr_at_20_diff1 + value: 46.75 + - type: nauc_mrr_at_100_max + value: 45.3233 + - type: nauc_mrr_at_100_std + value: -2.0778000000000003 + - type: nauc_mrr_at_100_diff1 + value: 46.7919 + - type: nauc_mrr_at_1000_max + value: 45.325700000000005 + - type: nauc_mrr_at_1000_std + value: -2.0868 + - type: nauc_mrr_at_1000_diff1 + value: 46.812799999999996 + - type: main_score + value: 53.047999999999995 task: type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackEnglishRetrieval (default) + revision: ad9991cb51e31e31e430383c75ffb2885547b5f0 + split: test + type: mteb/cqadupstack-english metrics: - type: ndcg_at_1 - value: 0.46354 + value: 35.796 + - type: ndcg_at_3 + value: 40.036 + - type: ndcg_at_5 + value: 41.778 - type: ndcg_at_10 - value: 0.47229 + value: 43.868 + - type: ndcg_at_20 + value: 45.777 - type: ndcg_at_100 - value: 0.5525 + value: 48.771 - type: ndcg_at_1000 - value: 0.57648 - - type: ndcg_at_20 - value: 0.50606 - - type: ndcg_at_3 - value: 0.42538 - - type: ndcg_at_5 - value: 0.43717 + value: 51.001 + - type: map_at_1 + value: 28.177000000000003 + - type: map_at_3 + value: 35.445 + - type: map_at_5 + value: 36.976 + - type: map_at_10 + value: 38.25 + - type: map_at_20 + value: 38.981 + - type: map_at_100 + value: 39.585 + - type: map_at_1000 + value: 39.728 - type: recall_at_1 - value: 0.20787 + value: 28.177000000000003 + - type: recall_at_3 + value: 41.782000000000004 + - type: recall_at_5 + value: 46.861000000000004 - type: recall_at_10 - value: 0.54771 + value: 53.464 + - type: recall_at_20 + value: 60.621 - type: recall_at_100 - value: 0.80689 + value: 74.628 - type: recall_at_1000 - value: 0.94032 - - type: recall_at_20 - value: 0.63842 - - type: recall_at_3 - value: 0.36229 - - type: recall_at_5 - value: 0.44437 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (ja) - config: ja - split: dev + value: 88.839 + - type: precision_at_1 + value: 35.796 + - type: precision_at_3 + value: 19.639 + - type: precision_at_5 + value: 13.924 + - type: precision_at_10 + value: 8.439 + - type: precision_at_20 + value: 5.016 + - type: precision_at_100 + value: 1.394 + - type: precision_at_1000 + value: 0.189 + - type: mrr_at_1 + value: 35.7962 + - type: mrr_at_3 + value: 42.1019 + - type: mrr_at_5 + value: 43.4172 + - type: mrr_at_10 + value: 44.2407 + - type: mrr_at_20 + value: 44.6907 + - type: mrr_at_100 + value: 45.0075 + - type: mrr_at_1000 + value: 45.059 + - type: nauc_ndcg_at_1_max + value: 47.856 + - type: nauc_ndcg_at_1_std + value: 3.0363 + - type: nauc_ndcg_at_1_diff1 + value: 48.7364 + - type: nauc_ndcg_at_3_max + value: 49.2728 + - type: nauc_ndcg_at_3_std + value: 4.1776 + - type: nauc_ndcg_at_3_diff1 + value: 45.1449 + - type: nauc_ndcg_at_5_max + value: 49.5649 + - type: nauc_ndcg_at_5_std + value: 3.7340999999999998 + - type: nauc_ndcg_at_5_diff1 + value: 44.6651 + - type: nauc_ndcg_at_10_max + value: 50.1977 + - type: nauc_ndcg_at_10_std + value: 4.5302 + - type: nauc_ndcg_at_10_diff1 + value: 45.0403 + - type: nauc_ndcg_at_20_max + value: 49.9326 + - type: nauc_ndcg_at_20_std + value: 5.5147 + - type: nauc_ndcg_at_20_diff1 + value: 44.5055 + - type: nauc_ndcg_at_100_max + value: 50.3035 + - type: nauc_ndcg_at_100_std + value: 7.1086 + - type: nauc_ndcg_at_100_diff1 + value: 44.451 + - type: nauc_ndcg_at_1000_max + value: 50.1836 + - type: nauc_ndcg_at_1000_std + value: 7.4503 + - type: nauc_ndcg_at_1000_diff1 + value: 44.301899999999996 + - type: nauc_map_at_1_max + value: 41.2555 + - type: nauc_map_at_1_std + value: -5.2668 + - type: nauc_map_at_1_diff1 + value: 52.0284 + - type: nauc_map_at_3_max + value: 46.6939 + - type: nauc_map_at_3_std + value: -0.8533000000000001 + - type: nauc_map_at_3_diff1 + value: 47.9095 + - type: nauc_map_at_5_max + value: 47.5024 + - type: nauc_map_at_5_std + value: -0.05109999999999999 + - type: nauc_map_at_5_diff1 + value: 47.1421 + - type: nauc_map_at_10_max + value: 48.1632 + - type: nauc_map_at_10_std + value: 0.8672 + - type: nauc_map_at_10_diff1 + value: 46.9929 + - type: nauc_map_at_20_max + value: 48.2708 + - type: nauc_map_at_20_std + value: 1.5195 + - type: nauc_map_at_20_diff1 + value: 46.7349 + - type: nauc_map_at_100_max + value: 48.5516 + - type: nauc_map_at_100_std + value: 2.1593 + - type: nauc_map_at_100_diff1 + value: 46.6641 + - type: nauc_map_at_1000_max + value: 48.6017 + - type: nauc_map_at_1000_std + value: 2.2745 + - type: nauc_map_at_1000_diff1 + value: 46.649 + - type: nauc_recall_at_1_max + value: 41.2555 + - type: nauc_recall_at_1_std + value: -5.2668 + - type: nauc_recall_at_1_diff1 + value: 52.0284 + - type: nauc_recall_at_3_max + value: 47.0403 + - type: nauc_recall_at_3_std + value: 1.5399 + - type: nauc_recall_at_3_diff1 + value: 42.998599999999996 + - type: nauc_recall_at_5_max + value: 47.7652 + - type: nauc_recall_at_5_std + value: 2.5079000000000002 + - type: nauc_recall_at_5_diff1 + value: 40.131099999999996 + - type: nauc_recall_at_10_max + value: 49.215199999999996 + - type: nauc_recall_at_10_std + value: 5.6207 + - type: nauc_recall_at_10_diff1 + value: 40.0067 + - type: nauc_recall_at_20_max + value: 47.6907 + - type: nauc_recall_at_20_std + value: 10.0091 + - type: nauc_recall_at_20_diff1 + value: 36.548 + - type: nauc_recall_at_100_max + value: 49.8978 + - type: nauc_recall_at_100_std + value: 20.7533 + - type: nauc_recall_at_100_diff1 + value: 34.463100000000004 + - type: nauc_recall_at_1000_max + value: 49.2751 + - type: nauc_recall_at_1000_std + value: 33.7021 + - type: nauc_recall_at_1000_diff1 + value: 27.995199999999997 + - type: nauc_precision_at_1_max + value: 47.856 + - type: nauc_precision_at_1_std + value: 3.0363 + - type: nauc_precision_at_1_diff1 + value: 48.7364 + - type: nauc_precision_at_3_max + value: 48.0591 + - type: nauc_precision_at_3_std + value: 16.0079 + - type: nauc_precision_at_3_diff1 + value: 28.286099999999998 + - type: nauc_precision_at_5_max + value: 45.3901 + - type: nauc_precision_at_5_std + value: 18.939500000000002 + - type: nauc_precision_at_5_diff1 + value: 20.7183 + - type: nauc_precision_at_10_max + value: 40.2901 + - type: nauc_precision_at_10_std + value: 24.1368 + - type: nauc_precision_at_10_diff1 + value: 13.1708 + - type: nauc_precision_at_20_max + value: 34.5736 + - type: nauc_precision_at_20_std + value: 28.524 + - type: nauc_precision_at_20_diff1 + value: 6.0857 + - type: nauc_precision_at_100_max + value: 24.0575 + - type: nauc_precision_at_100_std + value: 32.7048 + - type: nauc_precision_at_100_diff1 + value: -4.175800000000001 + - type: nauc_precision_at_1000_max + value: 11.3804 + - type: nauc_precision_at_1000_std + value: 28.917700000000004 + - type: nauc_precision_at_1000_diff1 + value: -11.994100000000001 + - type: nauc_mrr_at_1_max + value: 47.856 + - type: nauc_mrr_at_1_std + value: 3.0363 + - type: nauc_mrr_at_1_diff1 + value: 48.7364 + - type: nauc_mrr_at_3_max + value: 50.048 + - type: nauc_mrr_at_3_std + value: 6.464300000000001 + - type: nauc_mrr_at_3_diff1 + value: 45.5115 + - type: nauc_mrr_at_5_max + value: 50.0947 + - type: nauc_mrr_at_5_std + value: 6.3483 + - type: nauc_mrr_at_5_diff1 + value: 44.8476 + - type: nauc_mrr_at_10_max + value: 50.244699999999995 + - type: nauc_mrr_at_10_std + value: 6.666900000000001 + - type: nauc_mrr_at_10_diff1 + value: 45.0222 + - type: nauc_mrr_at_20_max + value: 50.1332 + - type: nauc_mrr_at_20_std + value: 6.868200000000001 + - type: nauc_mrr_at_20_diff1 + value: 44.8895 + - type: nauc_mrr_at_100_max + value: 50.1173 + - type: nauc_mrr_at_100_std + value: 6.930600000000001 + - type: nauc_mrr_at_100_diff1 + value: 44.8887 + - type: nauc_mrr_at_1000_max + value: 50.11259999999999 + - type: nauc_mrr_at_1000_std + value: 6.923799999999999 + - type: nauc_mrr_at_1000_diff1 + value: 44.8928 + - type: main_score + value: 43.868 task: type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGamingRetrieval (default) + revision: 4885aa143210c98657558c04aaf3dc47cfb54340 + split: test + type: mteb/cqadupstack-gaming metrics: - type: ndcg_at_1 - value: 0.56279 + value: 43.448 + - type: ndcg_at_3 + value: 51.032999999999994 + - type: ndcg_at_5 + value: 53.73 - type: ndcg_at_10 - value: 0.6281 + value: 56.369 + - type: ndcg_at_20 + value: 58.167 - type: ndcg_at_100 - value: 0.67757 + value: 60.28 - type: ndcg_at_1000 - value: 0.68667 - - type: ndcg_at_20 - value: 0.6521 - - type: ndcg_at_3 - value: 0.56226 - - type: ndcg_at_5 - value: 0.5866 + value: 61.511 + - type: map_at_1 + value: 38.115 + - type: map_at_3 + value: 47.355999999999995 + - type: map_at_5 + value: 49.221 + - type: map_at_10 + value: 50.57000000000001 + - type: map_at_20 + value: 51.2 + - type: map_at_100 + value: 51.568999999999996 + - type: map_at_1000 + value: 51.627 - type: recall_at_1 - value: 0.36648 + value: 38.115 + - type: recall_at_3 + value: 55.733 + - type: recall_at_5 + value: 62.41100000000001 - type: recall_at_10 - value: 0.7496 + value: 70.11800000000001 + - type: recall_at_20 + value: 76.714 - type: recall_at_100 - value: 0.92461 + value: 87.071 - type: recall_at_1000 - value: 0.97827 - - type: recall_at_20 - value: 0.82326 - - type: recall_at_3 - value: 0.55845 - - type: recall_at_5 - value: 0.63854 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (ko) - config: ko - split: dev + value: 95.921 + - type: precision_at_1 + value: 43.448 + - type: precision_at_3 + value: 22.947 + - type: precision_at_5 + value: 15.799 + - type: precision_at_10 + value: 9.154 + - type: precision_at_20 + value: 5.141 + - type: precision_at_100 + value: 1.196 + - type: precision_at_1000 + value: 0.135 + - type: mrr_at_1 + value: 43.4483 + - type: mrr_at_3 + value: 51.3689 + - type: mrr_at_5 + value: 52.8955 + - type: mrr_at_10 + value: 53.809200000000004 + - type: mrr_at_20 + value: 54.224700000000006 + - type: mrr_at_100 + value: 54.4617 + - type: mrr_at_1000 + value: 54.49079999999999 + - type: nauc_ndcg_at_1_max + value: 41.9268 + - type: nauc_ndcg_at_1_std + value: -6.0252 + - type: nauc_ndcg_at_1_diff1 + value: 55.4978 + - type: nauc_ndcg_at_3_max + value: 43.5492 + - type: nauc_ndcg_at_3_std + value: -4.7010000000000005 + - type: nauc_ndcg_at_3_diff1 + value: 51.0898 + - type: nauc_ndcg_at_5_max + value: 44.7544 + - type: nauc_ndcg_at_5_std + value: -2.9584 + - type: nauc_ndcg_at_5_diff1 + value: 50.6481 + - type: nauc_ndcg_at_10_max + value: 45.2203 + - type: nauc_ndcg_at_10_std + value: -1.6934 + - type: nauc_ndcg_at_10_diff1 + value: 49.9874 + - type: nauc_ndcg_at_20_max + value: 45.002199999999995 + - type: nauc_ndcg_at_20_std + value: -0.9383 + - type: nauc_ndcg_at_20_diff1 + value: 49.666700000000006 + - type: nauc_ndcg_at_100_max + value: 45.448699999999995 + - type: nauc_ndcg_at_100_std + value: -0.1934 + - type: nauc_ndcg_at_100_diff1 + value: 50.0483 + - type: nauc_ndcg_at_1000_max + value: 45.3335 + - type: nauc_ndcg_at_1000_std + value: -0.42389999999999994 + - type: nauc_ndcg_at_1000_diff1 + value: 50.5614 + - type: nauc_map_at_1_max + value: 35.7022 + - type: nauc_map_at_1_std + value: -6.6763 + - type: nauc_map_at_1_diff1 + value: 54.848699999999994 + - type: nauc_map_at_3_max + value: 41.5987 + - type: nauc_map_at_3_std + value: -6.3043000000000005 + - type: nauc_map_at_3_diff1 + value: 52.058400000000006 + - type: nauc_map_at_5_max + value: 42.5887 + - type: nauc_map_at_5_std + value: -5.0012 + - type: nauc_map_at_5_diff1 + value: 51.804300000000005 + - type: nauc_map_at_10_max + value: 43.085 + - type: nauc_map_at_10_std + value: -4.1721 + - type: nauc_map_at_10_diff1 + value: 51.524499999999996 + - type: nauc_map_at_20_max + value: 43.185 + - type: nauc_map_at_20_std + value: -3.6862 + - type: nauc_map_at_20_diff1 + value: 51.4297 + - type: nauc_map_at_100_max + value: 43.3473 + - type: nauc_map_at_100_std + value: -3.4286999999999996 + - type: nauc_map_at_100_diff1 + value: 51.497099999999996 + - type: nauc_map_at_1000_max + value: 43.358799999999995 + - type: nauc_map_at_1000_std + value: -3.3894 + - type: nauc_map_at_1000_diff1 + value: 51.5155 + - type: nauc_recall_at_1_max + value: 35.7022 + - type: nauc_recall_at_1_std + value: -6.6763 + - type: nauc_recall_at_1_diff1 + value: 54.848699999999994 + - type: nauc_recall_at_3_max + value: 42.9096 + - type: nauc_recall_at_3_std + value: -5.9907 + - type: nauc_recall_at_3_diff1 + value: 47.407 + - type: nauc_recall_at_5_max + value: 45.9891 + - type: nauc_recall_at_5_std + value: -0.5341 + - type: nauc_recall_at_5_diff1 + value: 45.336 + - type: nauc_recall_at_10_max + value: 47.457899999999995 + - type: nauc_recall_at_10_std + value: 4.2982 + - type: nauc_recall_at_10_diff1 + value: 41.6 + - type: nauc_recall_at_20_max + value: 47.3364 + - type: nauc_recall_at_20_std + value: 9.667100000000001 + - type: nauc_recall_at_20_diff1 + value: 38.4822 + - type: nauc_recall_at_100_max + value: 52.0554 + - type: nauc_recall_at_100_std + value: 21.6585 + - type: nauc_recall_at_100_diff1 + value: 35.2361 + - type: nauc_recall_at_1000_max + value: 62.38590000000001 + - type: nauc_recall_at_1000_std + value: 42.5442 + - type: nauc_recall_at_1000_diff1 + value: 37.1857 + - type: nauc_precision_at_1_max + value: 41.9268 + - type: nauc_precision_at_1_std + value: -6.0252 + - type: nauc_precision_at_1_diff1 + value: 55.4978 + - type: nauc_precision_at_3_max + value: 44.0934 + - type: nauc_precision_at_3_std + value: 2.4657 + - type: nauc_precision_at_3_diff1 + value: 33.468399999999995 + - type: nauc_precision_at_5_max + value: 41.8649 + - type: nauc_precision_at_5_std + value: 8.4992 + - type: nauc_precision_at_5_diff1 + value: 25.8132 + - type: nauc_precision_at_10_max + value: 36.8909 + - type: nauc_precision_at_10_std + value: 15.173200000000001 + - type: nauc_precision_at_10_diff1 + value: 16.0022 + - type: nauc_precision_at_20_max + value: 31.3774 + - type: nauc_precision_at_20_std + value: 21.304100000000002 + - type: nauc_precision_at_20_diff1 + value: 7.8406 + - type: nauc_precision_at_100_max + value: 23.828 + - type: nauc_precision_at_100_std + value: 27.3387 + - type: nauc_precision_at_100_diff1 + value: -0.5574 + - type: nauc_precision_at_1000_max + value: 14.3787 + - type: nauc_precision_at_1000_std + value: 27.8714 + - type: nauc_precision_at_1000_diff1 + value: -6.372400000000001 + - type: nauc_mrr_at_1_max + value: 41.9268 + - type: nauc_mrr_at_1_std + value: -6.0252 + - type: nauc_mrr_at_1_diff1 + value: 55.4978 + - type: nauc_mrr_at_3_max + value: 44.3228 + - type: nauc_mrr_at_3_std + value: -4.8039 + - type: nauc_mrr_at_3_diff1 + value: 52.6895 + - type: nauc_mrr_at_5_max + value: 45.0053 + - type: nauc_mrr_at_5_std + value: -3.5381000000000005 + - type: nauc_mrr_at_5_diff1 + value: 52.321 + - type: nauc_mrr_at_10_max + value: 44.9242 + - type: nauc_mrr_at_10_std + value: -3.2841 + - type: nauc_mrr_at_10_diff1 + value: 52.0518 + - type: nauc_mrr_at_20_max + value: 44.8189 + - type: nauc_mrr_at_20_std + value: -3.1717000000000004 + - type: nauc_mrr_at_20_diff1 + value: 52.0415 + - type: nauc_mrr_at_100_max + value: 44.8679 + - type: nauc_mrr_at_100_std + value: -3.1606 + - type: nauc_mrr_at_100_diff1 + value: 52.1083 + - type: nauc_mrr_at_1000_max + value: 44.864599999999996 + - type: nauc_mrr_at_1000_std + value: -3.167 + - type: nauc_mrr_at_1000_diff1 + value: 52.121399999999994 + - type: main_score + value: 56.369 task: type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackGisRetrieval (default) + revision: 5003b3064772da1887988e05400cf3806fe491f2 + split: test + type: mteb/cqadupstack-gis metrics: - type: ndcg_at_1 - value: 0.52582 + value: 31.863999999999997 + - type: ndcg_at_3 + value: 38.537 + - type: ndcg_at_5 + value: 41.104 - type: ndcg_at_10 - value: 0.59216 + value: 43.503 + - type: ndcg_at_20 + value: 45.413 - type: ndcg_at_100 - value: 0.65093 + value: 48.291000000000004 - type: ndcg_at_1000 - value: 0.66204 - - type: ndcg_at_20 - value: 0.62427 - - type: ndcg_at_3 - value: 0.5373 - - type: ndcg_at_5 - value: 0.55886 + value: 50.26199999999999 + - type: map_at_1 + value: 29.37 + - type: map_at_3 + value: 35.824 + - type: map_at_5 + value: 37.408 + - type: map_at_10 + value: 38.452999999999996 + - type: map_at_20 + value: 39.004 + - type: map_at_100 + value: 39.421 + - type: map_at_1000 + value: 39.501 - type: recall_at_1 - value: 0.30521 + value: 29.37 + - type: recall_at_3 + value: 43.442 + - type: recall_at_5 + value: 49.551 - type: recall_at_10 - value: 0.71159 + value: 56.791000000000004 + - type: recall_at_20 + value: 63.93 - type: recall_at_100 - value: 0.90203 + value: 78.666 - type: recall_at_1000 - value: 0.96714 - - type: recall_at_20 - value: 0.80209 - - type: recall_at_3 - value: 0.515 - - type: recall_at_5 - value: 0.6071 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (ru) - config: ru - split: dev + value: 93.354 + - type: precision_at_1 + value: 31.863999999999997 + - type: precision_at_3 + value: 16.083 + - type: precision_at_5 + value: 11.254 + - type: precision_at_10 + value: 6.508 + - type: precision_at_20 + value: 3.712 + - type: precision_at_100 + value: 0.9390000000000001 + - type: precision_at_1000 + value: 0.11399999999999999 + - type: mrr_at_1 + value: 31.8644 + - type: mrr_at_3 + value: 38.5122 + - type: mrr_at_5 + value: 39.873799999999996 + - type: mrr_at_10 + value: 40.8308 + - type: mrr_at_20 + value: 41.3284 + - type: mrr_at_100 + value: 41.6819 + - type: mrr_at_1000 + value: 41.7416 + - type: nauc_ndcg_at_1_max + value: 33.7601 + - type: nauc_ndcg_at_1_std + value: -9.8717 + - type: nauc_ndcg_at_1_diff1 + value: 42.2537 + - type: nauc_ndcg_at_3_max + value: 34.409600000000005 + - type: nauc_ndcg_at_3_std + value: -10.6027 + - type: nauc_ndcg_at_3_diff1 + value: 40.0317 + - type: nauc_ndcg_at_5_max + value: 34.0482 + - type: nauc_ndcg_at_5_std + value: -9.0778 + - type: nauc_ndcg_at_5_diff1 + value: 39.421499999999995 + - type: nauc_ndcg_at_10_max + value: 34.5365 + - type: nauc_ndcg_at_10_std + value: -7.3511999999999995 + - type: nauc_ndcg_at_10_diff1 + value: 38.6886 + - type: nauc_ndcg_at_20_max + value: 35.335699999999996 + - type: nauc_ndcg_at_20_std + value: -5.9596 + - type: nauc_ndcg_at_20_diff1 + value: 38.6051 + - type: nauc_ndcg_at_100_max + value: 34.6961 + - type: nauc_ndcg_at_100_std + value: -6.5812 + - type: nauc_ndcg_at_100_diff1 + value: 37.8079 + - type: nauc_ndcg_at_1000_max + value: 34.3938 + - type: nauc_ndcg_at_1000_std + value: -6.9155 + - type: nauc_ndcg_at_1000_diff1 + value: 38.2247 + - type: nauc_map_at_1_max + value: 32.231500000000004 + - type: nauc_map_at_1_std + value: -11.4991 + - type: nauc_map_at_1_diff1 + value: 44.7044 + - type: nauc_map_at_3_max + value: 34.0411 + - type: nauc_map_at_3_std + value: -10.8111 + - type: nauc_map_at_3_diff1 + value: 41.6004 + - type: nauc_map_at_5_max + value: 33.9275 + - type: nauc_map_at_5_std + value: -9.9881 + - type: nauc_map_at_5_diff1 + value: 41.1704 + - type: nauc_map_at_10_max + value: 34.1806 + - type: nauc_map_at_10_std + value: -9.2606 + - type: nauc_map_at_10_diff1 + value: 40.9213 + - type: nauc_map_at_20_max + value: 34.474 + - type: nauc_map_at_20_std + value: -8.798599999999999 + - type: nauc_map_at_20_diff1 + value: 40.9088 + - type: nauc_map_at_100_max + value: 34.381699999999995 + - type: nauc_map_at_100_std + value: -8.869 + - type: nauc_map_at_100_diff1 + value: 40.7894 + - type: nauc_map_at_1000_max + value: 34.3718 + - type: nauc_map_at_1000_std + value: -8.8674 + - type: nauc_map_at_1000_diff1 + value: 40.801700000000004 + - type: nauc_recall_at_1_max + value: 32.231500000000004 + - type: nauc_recall_at_1_std + value: -11.4991 + - type: nauc_recall_at_1_diff1 + value: 44.7044 + - type: nauc_recall_at_3_max + value: 33.4997 + - type: nauc_recall_at_3_std + value: -10.793999999999999 + - type: nauc_recall_at_3_diff1 + value: 36.8971 + - type: nauc_recall_at_5_max + value: 33.217600000000004 + - type: nauc_recall_at_5_std + value: -7.4771 + - type: nauc_recall_at_5_diff1 + value: 35.7378 + - type: nauc_recall_at_10_max + value: 34.3881 + - type: nauc_recall_at_10_std + value: -1.9206 + - type: nauc_recall_at_10_diff1 + value: 33.024300000000004 + - type: nauc_recall_at_20_max + value: 37.1734 + - type: nauc_recall_at_20_std + value: 4.5757 + - type: nauc_recall_at_20_diff1 + value: 31.7119 + - type: nauc_recall_at_100_max + value: 33.3328 + - type: nauc_recall_at_100_std + value: 4.0235 + - type: nauc_recall_at_100_diff1 + value: 23.5836 + - type: nauc_recall_at_1000_max + value: 23.6203 + - type: nauc_recall_at_1000_std + value: 10.4212 + - type: nauc_recall_at_1000_diff1 + value: 16.5204 + - type: nauc_precision_at_1_max + value: 33.7601 + - type: nauc_precision_at_1_std + value: -9.8717 + - type: nauc_precision_at_1_diff1 + value: 42.2537 + - type: nauc_precision_at_3_max + value: 37.046099999999996 + - type: nauc_precision_at_3_std + value: -8.1696 + - type: nauc_precision_at_3_diff1 + value: 32.893699999999995 + - type: nauc_precision_at_5_max + value: 33.5411 + - type: nauc_precision_at_5_std + value: -3.8621000000000003 + - type: nauc_precision_at_5_diff1 + value: 28.4192 + - type: nauc_precision_at_10_max + value: 33.8177 + - type: nauc_precision_at_10_std + value: 1.4605 + - type: nauc_precision_at_10_diff1 + value: 23.8779 + - type: nauc_precision_at_20_max + value: 33.2362 + - type: nauc_precision_at_20_std + value: 6.8675 + - type: nauc_precision_at_20_diff1 + value: 19.12 + - type: nauc_precision_at_100_max + value: 22.0581 + - type: nauc_precision_at_100_std + value: 5.6537999999999995 + - type: nauc_precision_at_100_diff1 + value: 2.677 + - type: nauc_precision_at_1000_max + value: 6.4192 + - type: nauc_precision_at_1000_std + value: 5.2604999999999995 + - type: nauc_precision_at_1000_diff1 + value: -12.5191 + - type: nauc_mrr_at_1_max + value: 33.7601 + - type: nauc_mrr_at_1_std + value: -9.8717 + - type: nauc_mrr_at_1_diff1 + value: 42.2537 + - type: nauc_mrr_at_3_max + value: 34.590700000000005 + - type: nauc_mrr_at_3_std + value: -9.3063 + - type: nauc_mrr_at_3_diff1 + value: 39.157599999999995 + - type: nauc_mrr_at_5_max + value: 34.262 + - type: nauc_mrr_at_5_std + value: -8.6629 + - type: nauc_mrr_at_5_diff1 + value: 38.7425 + - type: nauc_mrr_at_10_max + value: 34.3456 + - type: nauc_mrr_at_10_std + value: -8.0433 + - type: nauc_mrr_at_10_diff1 + value: 38.474199999999996 + - type: nauc_mrr_at_20_max + value: 34.504400000000004 + - type: nauc_mrr_at_20_std + value: -7.7764 + - type: nauc_mrr_at_20_diff1 + value: 38.4646 + - type: nauc_mrr_at_100_max + value: 34.407700000000006 + - type: nauc_mrr_at_100_std + value: -7.8669 + - type: nauc_mrr_at_100_diff1 + value: 38.4062 + - type: nauc_mrr_at_1000_max + value: 34.400999999999996 + - type: nauc_mrr_at_1000_std + value: -7.8653 + - type: nauc_mrr_at_1000_diff1 + value: 38.4264 + - type: main_score + value: 43.503 task: type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackMathematicaRetrieval (default) + revision: 90fceea13679c63fe563ded68f3b6f06e50061de + split: test + type: mteb/cqadupstack-mathematica metrics: - type: ndcg_at_1 - value: 0.47524 + value: 22.637 + - type: ndcg_at_3 + value: 26.865 + - type: ndcg_at_5 + value: 29.506 - type: ndcg_at_10 - value: 0.52349 + value: 32.024 + - type: ndcg_at_20 + value: 34.123999999999995 - type: ndcg_at_100 - value: 0.59725 + value: 38.013999999999996 - type: ndcg_at_1000 - value: 0.61313 - - type: ndcg_at_20 - value: 0.55669 - - type: ndcg_at_3 - value: 0.46812 - - type: ndcg_at_5 - value: 0.48442 + value: 40.681 + - type: map_at_1 + value: 18.354 + - type: map_at_3 + value: 23.777 + - type: map_at_5 + value: 25.380000000000003 + - type: map_at_10 + value: 26.588 + - type: map_at_20 + value: 27.227 + - type: map_at_100 + value: 27.851 + - type: map_at_1000 + value: 27.971 - type: recall_at_1 - value: 0.24337 + value: 18.354 + - type: recall_at_3 + value: 30.029 + - type: recall_at_5 + value: 36.716 - type: recall_at_10 - value: 0.62437 + value: 44.083 + - type: recall_at_20 + value: 51.653000000000006 - type: recall_at_100 - value: 0.86489 + value: 70.24000000000001 - type: recall_at_1000 - value: 0.95266 - - type: recall_at_20 - value: 0.71411 - - type: recall_at_3 - value: 0.42927 - - type: recall_at_5 - value: 0.51258 + value: 88.941 + - type: precision_at_1 + value: 22.637 + - type: precision_at_3 + value: 12.852 + - type: precision_at_5 + value: 9.652 + - type: precision_at_10 + value: 5.970000000000001 + - type: precision_at_20 + value: 3.557 + - type: precision_at_100 + value: 1.035 + - type: precision_at_1000 + value: 0.13899999999999998 + - type: mrr_at_1 + value: 22.6368 + - type: mrr_at_3 + value: 28.296 + - type: mrr_at_5 + value: 30.198999999999998 + - type: mrr_at_10 + value: 31.2411 + - type: mrr_at_20 + value: 31.773600000000002 + - type: mrr_at_100 + value: 32.230199999999996 + - type: mrr_at_1000 + value: 32.2949 + - type: nauc_ndcg_at_1_max + value: 31.0579 + - type: nauc_ndcg_at_1_std + value: -1.1154000000000002 + - type: nauc_ndcg_at_1_diff1 + value: 37.0188 + - type: nauc_ndcg_at_3_max + value: 30.6319 + - type: nauc_ndcg_at_3_std + value: 1.2079 + - type: nauc_ndcg_at_3_diff1 + value: 29.7055 + - type: nauc_ndcg_at_5_max + value: 29.2059 + - type: nauc_ndcg_at_5_std + value: 3.0105 + - type: nauc_ndcg_at_5_diff1 + value: 28.0947 + - type: nauc_ndcg_at_10_max + value: 29.2307 + - type: nauc_ndcg_at_10_std + value: 3.1515 + - type: nauc_ndcg_at_10_diff1 + value: 27.2115 + - type: nauc_ndcg_at_20_max + value: 29.1914 + - type: nauc_ndcg_at_20_std + value: 3.9833 + - type: nauc_ndcg_at_20_diff1 + value: 27.287899999999997 + - type: nauc_ndcg_at_100_max + value: 30.759999999999998 + - type: nauc_ndcg_at_100_std + value: 5.6163 + - type: nauc_ndcg_at_100_diff1 + value: 28.1445 + - type: nauc_ndcg_at_1000_max + value: 30.4012 + - type: nauc_ndcg_at_1000_std + value: 4.8586 + - type: nauc_ndcg_at_1000_diff1 + value: 27.7366 + - type: nauc_map_at_1_max + value: 26.9538 + - type: nauc_map_at_1_std + value: -0.9815 + - type: nauc_map_at_1_diff1 + value: 35.1964 + - type: nauc_map_at_3_max + value: 28.9516 + - type: nauc_map_at_3_std + value: 0.6373 + - type: nauc_map_at_3_diff1 + value: 30.476599999999998 + - type: nauc_map_at_5_max + value: 28.3735 + - type: nauc_map_at_5_std + value: 1.5893000000000002 + - type: nauc_map_at_5_diff1 + value: 29.4822 + - type: nauc_map_at_10_max + value: 28.4489 + - type: nauc_map_at_10_std + value: 1.7179 + - type: nauc_map_at_10_diff1 + value: 29.0721 + - type: nauc_map_at_20_max + value: 28.6443 + - type: nauc_map_at_20_std + value: 1.9567999999999999 + - type: nauc_map_at_20_diff1 + value: 29.2744 + - type: nauc_map_at_100_max + value: 28.9144 + - type: nauc_map_at_100_std + value: 2.2790999999999997 + - type: nauc_map_at_100_diff1 + value: 29.3889 + - type: nauc_map_at_1000_max + value: 28.8827 + - type: nauc_map_at_1000_std + value: 2.2127999999999997 + - type: nauc_map_at_1000_diff1 + value: 29.367700000000003 + - type: nauc_recall_at_1_max + value: 26.9538 + - type: nauc_recall_at_1_std + value: -0.9815 + - type: nauc_recall_at_1_diff1 + value: 35.1964 + - type: nauc_recall_at_3_max + value: 29.2823 + - type: nauc_recall_at_3_std + value: 2.2192 + - type: nauc_recall_at_3_diff1 + value: 25.174400000000002 + - type: nauc_recall_at_5_max + value: 26.098300000000002 + - type: nauc_recall_at_5_std + value: 5.870100000000001 + - type: nauc_recall_at_5_diff1 + value: 21.5717 + - type: nauc_recall_at_10_max + value: 26.3965 + - type: nauc_recall_at_10_std + value: 5.9524 + - type: nauc_recall_at_10_diff1 + value: 19.2576 + - type: nauc_recall_at_20_max + value: 25.014799999999997 + - type: nauc_recall_at_20_std + value: 8.889800000000001 + - type: nauc_recall_at_20_diff1 + value: 18.2048 + - type: nauc_recall_at_100_max + value: 32.664100000000005 + - type: nauc_recall_at_100_std + value: 20.66 + - type: nauc_recall_at_100_diff1 + value: 20.7167 + - type: nauc_recall_at_1000_max + value: 32.7425 + - type: nauc_recall_at_1000_std + value: 31.798 + - type: nauc_recall_at_1000_diff1 + value: 6.1744 + - type: nauc_precision_at_1_max + value: 31.0579 + - type: nauc_precision_at_1_std + value: -1.1154000000000002 + - type: nauc_precision_at_1_diff1 + value: 37.0188 + - type: nauc_precision_at_3_max + value: 34.0041 + - type: nauc_precision_at_3_std + value: 2.759 + - type: nauc_precision_at_3_diff1 + value: 26.0113 + - type: nauc_precision_at_5_max + value: 31.591599999999996 + - type: nauc_precision_at_5_std + value: 7.019499999999999 + - type: nauc_precision_at_5_diff1 + value: 22.5517 + - type: nauc_precision_at_10_max + value: 28.9779 + - type: nauc_precision_at_10_std + value: 6.0112 + - type: nauc_precision_at_10_diff1 + value: 18.4627 + - type: nauc_precision_at_20_max + value: 27.2677 + - type: nauc_precision_at_20_std + value: 7.9853 + - type: nauc_precision_at_20_diff1 + value: 17.6528 + - type: nauc_precision_at_100_max + value: 23.8248 + - type: nauc_precision_at_100_std + value: 9.9215 + - type: nauc_precision_at_100_diff1 + value: 13.5355 + - type: nauc_precision_at_1000_max + value: 9.9312 + - type: nauc_precision_at_1000_std + value: 1.8778 + - type: nauc_precision_at_1000_diff1 + value: 3.6692 + - type: nauc_mrr_at_1_max + value: 31.0579 + - type: nauc_mrr_at_1_std + value: -1.1154000000000002 + - type: nauc_mrr_at_1_diff1 + value: 37.0188 + - type: nauc_mrr_at_3_max + value: 32.265100000000004 + - type: nauc_mrr_at_3_std + value: 0.4738 + - type: nauc_mrr_at_3_diff1 + value: 31.6965 + - type: nauc_mrr_at_5_max + value: 31.610100000000003 + - type: nauc_mrr_at_5_std + value: 1.693 + - type: nauc_mrr_at_5_diff1 + value: 31.2068 + - type: nauc_mrr_at_10_max + value: 31.593500000000002 + - type: nauc_mrr_at_10_std + value: 1.6910999999999998 + - type: nauc_mrr_at_10_diff1 + value: 30.988300000000002 + - type: nauc_mrr_at_20_max + value: 31.4229 + - type: nauc_mrr_at_20_std + value: 1.9178000000000002 + - type: nauc_mrr_at_20_diff1 + value: 30.911 + - type: nauc_mrr_at_100_max + value: 31.510500000000004 + - type: nauc_mrr_at_100_std + value: 1.9404000000000001 + - type: nauc_mrr_at_100_diff1 + value: 30.928499999999996 + - type: nauc_mrr_at_1000_max + value: 31.499899999999997 + - type: nauc_mrr_at_1000_std + value: 1.9026999999999998 + - type: nauc_mrr_at_1000_diff1 + value: 30.9234 + - type: main_score + value: 32.024 + task: + type: Retrieval - dataset: - type: miracl/mmteb-miracl - name: Miracl (sw) - config: sw - split: dev - task: - type: Retrieval + config: default + name: MTEB CQADupstackPhysicsRetrieval (default) + revision: 79531abbd1fb92d06c6d6315a0cbbbf5bb247ea4 + split: test + type: mteb/cqadupstack-physics metrics: - type: ndcg_at_1 - value: 0.5166 + value: 36.477 + - type: ndcg_at_3 + value: 41.9 + - type: ndcg_at_5 + value: 44.352000000000004 - type: ndcg_at_10 - value: 0.61271 + value: 47.316 + - type: ndcg_at_20 + value: 49.262 - type: ndcg_at_100 - value: 0.66099 + value: 52.5 - type: ndcg_at_1000 - value: 0.66867 - - type: ndcg_at_20 - value: 0.63643 + value: 54.433 + - type: map_at_1 + value: 29.633 + - type: map_at_3 + value: 37.374 + - type: map_at_5 + value: 39.327 + - type: map_at_10 + value: 40.897 + - type: map_at_20 + value: 41.629 + - type: map_at_100 + value: 42.221 + - type: map_at_1000 + value: 42.337 + - type: recall_at_1 + value: 29.633 + - type: recall_at_3 + value: 45.141999999999996 + - type: recall_at_5 + value: 51.578 + - type: recall_at_10 + value: 60.465999999999994 + - type: recall_at_20 + value: 67.012 + - type: recall_at_100 + value: 82.174 + - type: recall_at_1000 + value: 94.65 + - type: precision_at_1 + value: 36.477 + - type: precision_at_3 + value: 20.308 + - type: precision_at_5 + value: 14.379 + - type: precision_at_10 + value: 8.816 + - type: precision_at_20 + value: 5.106 + - type: precision_at_100 + value: 1.3419999999999999 + - type: precision_at_1000 + value: 0.169 + - type: mrr_at_1 + value: 36.477399999999996 + - type: mrr_at_3 + value: 44.0648 + - type: mrr_at_5 + value: 45.4604 + - type: mrr_at_10 + value: 46.6132 + - type: mrr_at_20 + value: 47.0122 + - type: mrr_at_100 + value: 47.3432 + - type: mrr_at_1000 + value: 47.383900000000004 + - type: nauc_ndcg_at_1_max + value: 44.2532 + - type: nauc_ndcg_at_1_std + value: 0.27399999999999997 + - type: nauc_ndcg_at_1_diff1 + value: 56.0608 + - type: nauc_ndcg_at_3_max + value: 40.7243 + - type: nauc_ndcg_at_3_std + value: -3.0545 + - type: nauc_ndcg_at_3_diff1 + value: 48.4101 + - type: nauc_ndcg_at_5_max + value: 39.556999999999995 + - type: nauc_ndcg_at_5_std + value: -3.9035 + - type: nauc_ndcg_at_5_diff1 + value: 47.2832 + - type: nauc_ndcg_at_10_max + value: 39.6116 + - type: nauc_ndcg_at_10_std + value: -4.2111 + - type: nauc_ndcg_at_10_diff1 + value: 47.0266 + - type: nauc_ndcg_at_20_max + value: 40.1775 + - type: nauc_ndcg_at_20_std + value: -2.9367 + - type: nauc_ndcg_at_20_diff1 + value: 47.4448 + - type: nauc_ndcg_at_100_max + value: 41.9972 + - type: nauc_ndcg_at_100_std + value: 0.46740000000000004 + - type: nauc_ndcg_at_100_diff1 + value: 48.4355 + - type: nauc_ndcg_at_1000_max + value: 42.1182 + - type: nauc_ndcg_at_1000_std + value: 0.8456 + - type: nauc_ndcg_at_1000_diff1 + value: 48.1614 + - type: nauc_map_at_1_max + value: 37.5422 + - type: nauc_map_at_1_std + value: -4.2909999999999995 + - type: nauc_map_at_1_diff1 + value: 55.083800000000004 + - type: nauc_map_at_3_max + value: 39.0107 + - type: nauc_map_at_3_std + value: -4.3038 + - type: nauc_map_at_3_diff1 + value: 49.5355 + - type: nauc_map_at_5_max + value: 38.9933 + - type: nauc_map_at_5_std + value: -4.3489 + - type: nauc_map_at_5_diff1 + value: 48.9543 + - type: nauc_map_at_10_max + value: 39.2673 + - type: nauc_map_at_10_std + value: -4.1611 + - type: nauc_map_at_10_diff1 + value: 48.891400000000004 + - type: nauc_map_at_20_max + value: 39.533699999999996 + - type: nauc_map_at_20_std + value: -3.7303 + - type: nauc_map_at_20_diff1 + value: 49.001099999999994 + - type: nauc_map_at_100_max + value: 39.9274 + - type: nauc_map_at_100_std + value: -3.0797000000000003 + - type: nauc_map_at_100_diff1 + value: 49.1862 + - type: nauc_map_at_1000_max + value: 39.957100000000004 + - type: nauc_map_at_1000_std + value: -3.0084 + - type: nauc_map_at_1000_diff1 + value: 49.1595 + - type: nauc_recall_at_1_max + value: 37.5422 + - type: nauc_recall_at_1_std + value: -4.2909999999999995 + - type: nauc_recall_at_1_diff1 + value: 55.083800000000004 + - type: nauc_recall_at_3_max + value: 35.5355 + - type: nauc_recall_at_3_std + value: -7.140000000000001 + - type: nauc_recall_at_3_diff1 + value: 42.4278 + - type: nauc_recall_at_5_max + value: 33.9238 + - type: nauc_recall_at_5_std + value: -7.9919 + - type: nauc_recall_at_5_diff1 + value: 39.1808 + - type: nauc_recall_at_10_max + value: 33.4493 + - type: nauc_recall_at_10_std + value: -9.1861 + - type: nauc_recall_at_10_diff1 + value: 36.8475 + - type: nauc_recall_at_20_max + value: 34.9121 + - type: nauc_recall_at_20_std + value: -4.8026 + - type: nauc_recall_at_20_diff1 + value: 37.9247 + - type: nauc_recall_at_100_max + value: 44.1541 + - type: nauc_recall_at_100_std + value: 18.1134 + - type: nauc_recall_at_100_diff1 + value: 41.6633 + - type: nauc_recall_at_1000_max + value: 56.3385 + - type: nauc_recall_at_1000_std + value: 53.257299999999994 + - type: nauc_recall_at_1000_diff1 + value: 36.1232 + - type: nauc_precision_at_1_max + value: 44.2532 + - type: nauc_precision_at_1_std + value: 0.27399999999999997 + - type: nauc_precision_at_1_diff1 + value: 56.0608 + - type: nauc_precision_at_3_max + value: 41.179 + - type: nauc_precision_at_3_std + value: 5.588 + - type: nauc_precision_at_3_diff1 + value: 32.8574 + - type: nauc_precision_at_5_max + value: 34.808699999999995 + - type: nauc_precision_at_5_std + value: 6.261 + - type: nauc_precision_at_5_diff1 + value: 23.993100000000002 + - type: nauc_precision_at_10_max + value: 30.966500000000003 + - type: nauc_precision_at_10_std + value: 9.9887 + - type: nauc_precision_at_10_diff1 + value: 16.8352 + - type: nauc_precision_at_20_max + value: 26.977600000000002 + - type: nauc_precision_at_20_std + value: 14.0043 + - type: nauc_precision_at_20_diff1 + value: 10.9725 + - type: nauc_precision_at_100_max + value: 20.0541 + - type: nauc_precision_at_100_std + value: 24.0399 + - type: nauc_precision_at_100_diff1 + value: -0.46509999999999996 + - type: nauc_precision_at_1000_max + value: 8.1382 + - type: nauc_precision_at_1000_std + value: 21.7963 + - type: nauc_precision_at_1000_diff1 + value: -13.7289 + - type: nauc_mrr_at_1_max + value: 44.2532 + - type: nauc_mrr_at_1_std + value: 0.27399999999999997 + - type: nauc_mrr_at_1_diff1 + value: 56.0608 + - type: nauc_mrr_at_3_max + value: 43.0277 + - type: nauc_mrr_at_3_std + value: -0.8843 + - type: nauc_mrr_at_3_diff1 + value: 51.112899999999996 + - type: nauc_mrr_at_5_max + value: 42.852000000000004 + - type: nauc_mrr_at_5_std + value: -0.8572 + - type: nauc_mrr_at_5_diff1 + value: 50.4937 + - type: nauc_mrr_at_10_max + value: 43.0093 + - type: nauc_mrr_at_10_std + value: -0.8631 + - type: nauc_mrr_at_10_diff1 + value: 50.41909999999999 + - type: nauc_mrr_at_20_max + value: 43.0484 + - type: nauc_mrr_at_20_std + value: -0.6054999999999999 + - type: nauc_mrr_at_20_diff1 + value: 50.527100000000004 + - type: nauc_mrr_at_100_max + value: 43.175200000000004 + - type: nauc_mrr_at_100_std + value: -0.3019 + - type: nauc_mrr_at_100_diff1 + value: 50.5962 + - type: nauc_mrr_at_1000_max + value: 43.173899999999996 + - type: nauc_mrr_at_1000_std + value: -0.3115 + - type: nauc_mrr_at_1000_diff1 + value: 50.6012 + - type: main_score + value: 47.316 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackProgrammersRetrieval (default) + revision: 6184bc1440d2dbc7612be22b50686b8826d22b32 + split: test + type: mteb/cqadupstack-programmers + metrics: + - type: ndcg_at_1 + value: 33.676 - type: ndcg_at_3 - value: 0.54828 + value: 38.7 - type: ndcg_at_5 - value: 0.57382 + value: 41.032999999999994 + - type: ndcg_at_10 + value: 43.580999999999996 + - type: ndcg_at_20 + value: 45.992 + - type: ndcg_at_100 + value: 49.192 + - type: ndcg_at_1000 + value: 51.473 + - type: map_at_1 + value: 27.389999999999997 + - type: map_at_3 + value: 34.660999999999994 + - type: map_at_5 + value: 36.38 + - type: map_at_10 + value: 37.768 + - type: map_at_20 + value: 38.534 + - type: map_at_100 + value: 39.091 + - type: map_at_1000 + value: 39.2 - type: recall_at_1 - value: 0.35277 + value: 27.389999999999997 + - type: recall_at_3 + value: 41.876000000000005 + - type: recall_at_5 + value: 47.961999999999996 - type: recall_at_10 - value: 0.74368 + value: 55.445 + - type: recall_at_20 + value: 64.143 - type: recall_at_100 - value: 0.92261 + value: 79.327 - type: recall_at_1000 - value: 0.97109 - - type: recall_at_20 - value: 0.81888 + value: 94.64200000000001 + - type: precision_at_1 + value: 33.676 + - type: precision_at_3 + value: 18.455 + - type: precision_at_5 + value: 13.128 + - type: precision_at_10 + value: 7.888000000000001 + - type: precision_at_20 + value: 4.697 + - type: precision_at_100 + value: 1.234 + - type: precision_at_1000 + value: 0.161 + - type: mrr_at_1 + value: 33.6758 + - type: mrr_at_3 + value: 40.7725 + - type: mrr_at_5 + value: 42.267900000000004 + - type: mrr_at_10 + value: 43.1813 + - type: mrr_at_20 + value: 43.769200000000005 + - type: mrr_at_100 + value: 44.0965 + - type: mrr_at_1000 + value: 44.149899999999995 + - type: nauc_ndcg_at_1_max + value: 47.957699999999996 + - type: nauc_ndcg_at_1_std + value: 11.211 + - type: nauc_ndcg_at_1_diff1 + value: 50.975899999999996 + - type: nauc_ndcg_at_3_max + value: 46.7077 + - type: nauc_ndcg_at_3_std + value: 11.8166 + - type: nauc_ndcg_at_3_diff1 + value: 44.183699999999995 + - type: nauc_ndcg_at_5_max + value: 46.5691 + - type: nauc_ndcg_at_5_std + value: 12.3224 + - type: nauc_ndcg_at_5_diff1 + value: 43.2912 + - type: nauc_ndcg_at_10_max + value: 45.989200000000004 + - type: nauc_ndcg_at_10_std + value: 13.4501 + - type: nauc_ndcg_at_10_diff1 + value: 41.3206 + - type: nauc_ndcg_at_20_max + value: 46.400400000000005 + - type: nauc_ndcg_at_20_std + value: 15.004000000000001 + - type: nauc_ndcg_at_20_diff1 + value: 40.8932 + - type: nauc_ndcg_at_100_max + value: 47.3346 + - type: nauc_ndcg_at_100_std + value: 16.5132 + - type: nauc_ndcg_at_100_diff1 + value: 42.126599999999996 + - type: nauc_ndcg_at_1000_max + value: 47.5217 + - type: nauc_ndcg_at_1000_std + value: 15.4551 + - type: nauc_ndcg_at_1000_diff1 + value: 42.5563 + - type: nauc_map_at_1_max + value: 42.549 + - type: nauc_map_at_1_std + value: 4.9833 + - type: nauc_map_at_1_diff1 + value: 52.14339999999999 + - type: nauc_map_at_3_max + value: 44.8114 + - type: nauc_map_at_3_std + value: 9.440800000000001 + - type: nauc_map_at_3_diff1 + value: 46.1197 + - type: nauc_map_at_5_max + value: 45.3059 + - type: nauc_map_at_5_std + value: 10.286900000000001 + - type: nauc_map_at_5_diff1 + value: 45.6263 + - type: nauc_map_at_10_max + value: 45.3517 + - type: nauc_map_at_10_std + value: 11.1304 + - type: nauc_map_at_10_diff1 + value: 44.6502 + - type: nauc_map_at_20_max + value: 45.5319 + - type: nauc_map_at_20_std + value: 11.5773 + - type: nauc_map_at_20_diff1 + value: 44.5681 + - type: nauc_map_at_100_max + value: 45.8019 + - type: nauc_map_at_100_std + value: 11.9772 + - type: nauc_map_at_100_diff1 + value: 44.7825 + - type: nauc_map_at_1000_max + value: 45.8134 + - type: nauc_map_at_1000_std + value: 11.9461 + - type: nauc_map_at_1000_diff1 + value: 44.7905 + - type: nauc_recall_at_1_max + value: 42.549 + - type: nauc_recall_at_1_std + value: 4.9833 + - type: nauc_recall_at_1_diff1 + value: 52.14339999999999 + - type: nauc_recall_at_3_max + value: 44.0409 + - type: nauc_recall_at_3_std + value: 11.9146 + - type: nauc_recall_at_3_diff1 + value: 38.6436 + - type: nauc_recall_at_5_max + value: 43.3961 + - type: nauc_recall_at_5_std + value: 12.6675 + - type: nauc_recall_at_5_diff1 + value: 35.5553 + - type: nauc_recall_at_10_max + value: 41.4966 + - type: nauc_recall_at_10_std + value: 16.1644 + - type: nauc_recall_at_10_diff1 + value: 29.2835 + - type: nauc_recall_at_20_max + value: 41.474 + - type: nauc_recall_at_20_std + value: 22.5684 + - type: nauc_recall_at_20_diff1 + value: 25.7308 + - type: nauc_recall_at_100_max + value: 45.1253 + - type: nauc_recall_at_100_std + value: 36.248799999999996 + - type: nauc_recall_at_100_diff1 + value: 28.799500000000002 + - type: nauc_recall_at_1000_max + value: 54.1747 + - type: nauc_recall_at_1000_std + value: 47.1501 + - type: nauc_recall_at_1000_diff1 + value: 23.198900000000002 + - type: nauc_precision_at_1_max + value: 47.957699999999996 + - type: nauc_precision_at_1_std + value: 11.211 + - type: nauc_precision_at_1_diff1 + value: 50.975899999999996 + - type: nauc_precision_at_3_max + value: 46.6181 + - type: nauc_precision_at_3_std + value: 19.475 + - type: nauc_precision_at_3_diff1 + value: 30.6784 + - type: nauc_precision_at_5_max + value: 43.5114 + - type: nauc_precision_at_5_std + value: 22.1293 + - type: nauc_precision_at_5_diff1 + value: 24.6525 + - type: nauc_precision_at_10_max + value: 37.47 + - type: nauc_precision_at_10_std + value: 23.8068 + - type: nauc_precision_at_10_diff1 + value: 14.9368 + - type: nauc_precision_at_20_max + value: 33.4529 + - type: nauc_precision_at_20_std + value: 25.4979 + - type: nauc_precision_at_20_diff1 + value: 9.4501 + - type: nauc_precision_at_100_max + value: 23.7406 + - type: nauc_precision_at_100_std + value: 22.8583 + - type: nauc_precision_at_100_diff1 + value: 3.6348 + - type: nauc_precision_at_1000_max + value: 4.5396 + - type: nauc_precision_at_1000_std + value: 6.0796 + - type: nauc_precision_at_1000_diff1 + value: -7.2498000000000005 + - type: nauc_mrr_at_1_max + value: 47.957699999999996 + - type: nauc_mrr_at_1_std + value: 11.211 + - type: nauc_mrr_at_1_diff1 + value: 50.975899999999996 + - type: nauc_mrr_at_3_max + value: 48.6226 + - type: nauc_mrr_at_3_std + value: 13.600000000000001 + - type: nauc_mrr_at_3_diff1 + value: 45.2881 + - type: nauc_mrr_at_5_max + value: 48.402499999999996 + - type: nauc_mrr_at_5_std + value: 13.616 + - type: nauc_mrr_at_5_diff1 + value: 44.7074 + - type: nauc_mrr_at_10_max + value: 48.0556 + - type: nauc_mrr_at_10_std + value: 13.7803 + - type: nauc_mrr_at_10_diff1 + value: 44.0852 + - type: nauc_mrr_at_20_max + value: 48.173500000000004 + - type: nauc_mrr_at_20_std + value: 14.1617 + - type: nauc_mrr_at_20_diff1 + value: 44.0396 + - type: nauc_mrr_at_100_max + value: 48.1841 + - type: nauc_mrr_at_100_std + value: 14.1827 + - type: nauc_mrr_at_100_diff1 + value: 44.210100000000004 + - type: nauc_mrr_at_1000_max + value: 48.1875 + - type: nauc_mrr_at_1000_std + value: 14.161000000000001 + - type: nauc_mrr_at_1000_diff1 + value: 44.222 + - type: main_score + value: 43.580999999999996 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackRetrieval (default) + revision: 160c094312a0e1facb97e55eeddb698c0abe3571 + split: test + type: CQADupstackRetrieval_is_a_combined_dataset + metrics: + - type: ndcg_at_1 + value: 32.588499999999996 + - type: ndcg_at_3 + value: 37.949083333333334 + - type: ndcg_at_5 + value: 40.258833333333335 + - type: ndcg_at_10 + value: 42.74341666666667 + - type: ndcg_at_20 + value: 44.784 + - type: ndcg_at_100 + value: 47.903416666666665 + - type: ndcg_at_1000 + value: 50.067416666666674 + - type: map_at_1 + value: 27.52808333333333 + - type: map_at_3 + value: 34.321999999999996 + - type: map_at_5 + value: 35.96091666666666 + - type: map_at_10 + value: 37.22708333333333 + - type: map_at_20 + value: 37.914833333333334 + - type: map_at_100 + value: 38.462166666666675 + - type: map_at_1000 + value: 38.57725 + - type: recall_at_1 + value: 27.52808333333333 - type: recall_at_3 - value: 0.56739 + value: 41.30075 - type: recall_at_5 - value: 0.6421 + value: 47.26408333333334 + - type: recall_at_10 + value: 54.663833333333336 + - type: recall_at_20 + value: 62.11658333333333 + - type: recall_at_100 + value: 77.176 + - type: recall_at_1000 + value: 92.03791666666666 + - type: precision_at_1 + value: 32.588499999999996 + - type: precision_at_3 + value: 17.485 + - type: precision_at_5 + value: 12.427666666666669 + - type: precision_at_10 + value: 7.493333333333334 + - type: precision_at_20 + value: 4.413499999999999 + - type: precision_at_100 + value: 1.18675 + - type: precision_at_1000 + value: 0.15691666666666665 + - type: mrr_at_1 + value: 32.58871666666667 + - type: mrr_at_3 + value: 39.09032499999999 + - type: mrr_at_5 + value: 40.533125 + - type: mrr_at_10 + value: 41.51483333333333 + - type: mrr_at_20 + value: 42.01036666666667 + - type: mrr_at_100 + value: 42.35724166666667 + - type: mrr_at_1000 + value: 42.41010833333333 + - type: nauc_ndcg_at_1_max + value: 41.86760833333334 + - type: nauc_ndcg_at_1_std + value: -0.022441666666666443 + - type: nauc_ndcg_at_1_diff1 + value: 48.604266666666675 + - type: nauc_ndcg_at_3_max + value: 40.649825 + - type: nauc_ndcg_at_3_std + value: 0.9594416666666666 + - type: nauc_ndcg_at_3_diff1 + value: 42.754375 + - type: nauc_ndcg_at_5_max + value: 40.71646666666666 + - type: nauc_ndcg_at_5_std + value: 1.8118249999999998 + - type: nauc_ndcg_at_5_diff1 + value: 42.09031666666666 + - type: nauc_ndcg_at_10_max + value: 40.616033333333334 + - type: nauc_ndcg_at_10_std + value: 2.621475 + - type: nauc_ndcg_at_10_diff1 + value: 41.56405833333333 + - type: nauc_ndcg_at_20_max + value: 41.00335 + - type: nauc_ndcg_at_20_std + value: 3.5835 + - type: nauc_ndcg_at_20_diff1 + value: 41.526025 + - type: nauc_ndcg_at_100_max + value: 41.626575 + - type: nauc_ndcg_at_100_std + value: 4.921058333333334 + - type: nauc_ndcg_at_100_diff1 + value: 41.785700000000006 + - type: nauc_ndcg_at_1000_max + value: 41.623041666666666 + - type: nauc_ndcg_at_1000_std + value: 4.743416666666667 + - type: nauc_ndcg_at_1000_diff1 + value: 41.930049999999994 + - type: nauc_map_at_1_max + value: 37.757374999999996 + - type: nauc_map_at_1_std + value: -2.7256583333333335 + - type: nauc_map_at_1_diff1 + value: 49.68454166666667 + - type: nauc_map_at_3_max + value: 39.41603333333333 + - type: nauc_map_at_3_std + value: -0.7485333333333334 + - type: nauc_map_at_3_diff1 + value: 44.64258333333333 + - type: nauc_map_at_5_max + value: 39.84875833333333 + - type: nauc_map_at_5_std + value: 0.010733333333333428 + - type: nauc_map_at_5_diff1 + value: 44.133975 + - type: nauc_map_at_10_max + value: 40.05009166666666 + - type: nauc_map_at_10_std + value: 0.6503083333333333 + - type: nauc_map_at_10_diff1 + value: 43.826724999999996 + - type: nauc_map_at_20_max + value: 40.287733333333335 + - type: nauc_map_at_20_std + value: 1.0432333333333332 + - type: nauc_map_at_20_diff1 + value: 43.784241666666674 + - type: nauc_map_at_100_max + value: 40.44630833333334 + - type: nauc_map_at_100_std + value: 1.3809583333333333 + - type: nauc_map_at_100_diff1 + value: 43.81610833333333 + - type: nauc_map_at_1000_max + value: 40.45624166666667 + - type: nauc_map_at_1000_std + value: 1.4088416666666665 + - type: nauc_map_at_1000_diff1 + value: 43.81260833333333 + - type: nauc_recall_at_1_max + value: 37.757374999999996 + - type: nauc_recall_at_1_std + value: -2.7256583333333335 + - type: nauc_recall_at_1_diff1 + value: 49.68454166666667 + - type: nauc_recall_at_3_max + value: 37.99286666666667 + - type: nauc_recall_at_3_std + value: 0.5074666666666666 + - type: nauc_recall_at_3_diff1 + value: 38.458816666666664 + - type: nauc_recall_at_5_max + value: 38.23744166666667 + - type: nauc_recall_at_5_std + value: 2.8538000000000006 + - type: nauc_recall_at_5_diff1 + value: 36.16175833333334 + - type: nauc_recall_at_10_max + value: 37.54170833333333 + - type: nauc_recall_at_10_std + value: 5.354441666666667 + - type: nauc_recall_at_10_diff1 + value: 33.80731666666667 + - type: nauc_recall_at_20_max + value: 38.071758333333335 + - type: nauc_recall_at_20_std + value: 9.4403 + - type: nauc_recall_at_20_diff1 + value: 32.409758333333336 + - type: nauc_recall_at_100_max + value: 41.127158333333334 + - type: nauc_recall_at_100_std + value: 20.718875000000004 + - type: nauc_recall_at_100_diff1 + value: 30.971016666666664 + - type: nauc_recall_at_1000_max + value: 44.978608333333334 + - type: nauc_recall_at_1000_std + value: 39.36581666666667 + - type: nauc_recall_at_1000_diff1 + value: 27.076241666666668 + - type: nauc_precision_at_1_max + value: 41.86760833333334 + - type: nauc_precision_at_1_std + value: -0.022441666666666443 + - type: nauc_precision_at_1_diff1 + value: 48.604266666666675 + - type: nauc_precision_at_3_max + value: 40.53820000000001 + - type: nauc_precision_at_3_std + value: 6.682866666666667 + - type: nauc_precision_at_3_diff1 + value: 30.627458333333337 + - type: nauc_precision_at_5_max + value: 38.085708333333336 + - type: nauc_precision_at_5_std + value: 10.236816666666666 + - type: nauc_precision_at_5_diff1 + value: 24.589866666666666 + - type: nauc_precision_at_10_max + value: 33.795766666666665 + - type: nauc_precision_at_10_std + value: 13.644358333333335 + - type: nauc_precision_at_10_diff1 + value: 17.663875 + - type: nauc_precision_at_20_max + value: 30.67170833333333 + - type: nauc_precision_at_20_std + value: 16.899591666666666 + - type: nauc_precision_at_20_diff1 + value: 12.398666666666665 + - type: nauc_precision_at_100_max + value: 21.46699166666666 + - type: nauc_precision_at_100_std + value: 19.683266666666665 + - type: nauc_precision_at_100_diff1 + value: 2.3721666666666668 + - type: nauc_precision_at_1000_max + value: 6.773875 + - type: nauc_precision_at_1000_std + value: 13.712933333333336 + - type: nauc_precision_at_1000_diff1 + value: -9.302758333333333 + - type: nauc_mrr_at_1_max + value: 41.86760833333334 + - type: nauc_mrr_at_1_std + value: -0.022441666666666443 + - type: nauc_mrr_at_1_diff1 + value: 48.604266666666675 + - type: nauc_mrr_at_3_max + value: 42.065525 + - type: nauc_mrr_at_3_std + value: 1.6751166666666664 + - type: nauc_mrr_at_3_diff1 + value: 43.90220833333333 + - type: nauc_mrr_at_5_max + value: 42.07275833333333 + - type: nauc_mrr_at_5_std + value: 2.3014749999999995 + - type: nauc_mrr_at_5_diff1 + value: 43.440275 + - type: nauc_mrr_at_10_max + value: 41.955425000000005 + - type: nauc_mrr_at_10_std + value: 2.499491666666667 + - type: nauc_mrr_at_10_diff1 + value: 43.23685833333333 + - type: nauc_mrr_at_20_max + value: 41.98479166666666 + - type: nauc_mrr_at_20_std + value: 2.6983083333333333 + - type: nauc_mrr_at_20_diff1 + value: 43.24806666666667 + - type: nauc_mrr_at_100_max + value: 42.01090833333334 + - type: nauc_mrr_at_100_std + value: 2.7583083333333334 + - type: nauc_mrr_at_100_diff1 + value: 43.28899166666667 + - type: nauc_mrr_at_1000_max + value: 42.010841666666664 + - type: nauc_mrr_at_1000_std + value: 2.750433333333333 + - type: nauc_mrr_at_1000_diff1 + value: 43.299625 + - type: main_score + value: 42.74341666666667 + task: + type: Retrieval - dataset: - type: miracl/mmteb-miracl - name: Miracl (te) - config: te - split: dev + config: default + name: MTEB CQADupstackRetrieval (default) + revision: CQADupstackRetrieval_is_a_combined_dataset + split: test + type: CQADupstackRetrieval_is_a_combined_dataset + metrics: + - type: main_score + value: 42.743416666666675 + - type: ndcg_at_10 + value: 42.743416666666675 task: type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackStatsRetrieval (default) + revision: 65ac3a16b8e91f9cee4c9828cc7c335575432a2a + split: test + type: mteb/cqadupstack-stats metrics: - type: ndcg_at_1 - value: 0.63768 + value: 27.607 + - type: ndcg_at_3 + value: 32.665 + - type: ndcg_at_5 + value: 34.876000000000005 - type: ndcg_at_10 - value: 0.79193 + value: 36.796 + - type: ndcg_at_20 + value: 38.405 - type: ndcg_at_100 - value: 0.80243 + value: 41.612 - type: ndcg_at_1000 - value: 0.80438 - - type: ndcg_at_20 - value: 0.79549 + value: 43.869 + - type: map_at_1 + value: 24.748 + - type: map_at_3 + value: 30.192999999999998 + - type: map_at_5 + value: 31.563999999999997 + - type: map_at_10 + value: 32.424 + - type: map_at_20 + value: 32.905 + - type: map_at_100 + value: 33.385 + - type: map_at_1000 + value: 33.476 + - type: recall_at_1 + value: 24.748 + - type: recall_at_3 + value: 36.14 + - type: recall_at_5 + value: 41.617 + - type: recall_at_10 + value: 47.49 + - type: recall_at_20 + value: 53.413 + - type: recall_at_100 + value: 69.461 + - type: recall_at_1000 + value: 86.014 + - type: precision_at_1 + value: 27.607 + - type: precision_at_3 + value: 13.957 + - type: precision_at_5 + value: 9.847 + - type: precision_at_10 + value: 5.782 + - type: precision_at_20 + value: 3.3360000000000003 + - type: precision_at_100 + value: 0.906 + - type: precision_at_1000 + value: 0.11800000000000001 + - type: mrr_at_1 + value: 27.6074 + - type: mrr_at_3 + value: 32.9499 + - type: mrr_at_5 + value: 34.2229 + - type: mrr_at_10 + value: 35.0668 + - type: mrr_at_20 + value: 35.4859 + - type: mrr_at_100 + value: 35.8948 + - type: mrr_at_1000 + value: 35.9562 + - type: nauc_ndcg_at_1_max + value: 49.1944 + - type: nauc_ndcg_at_1_std + value: 11.7093 + - type: nauc_ndcg_at_1_diff1 + value: 56.8806 + - type: nauc_ndcg_at_3_max + value: 46.7361 + - type: nauc_ndcg_at_3_std + value: 13.4354 + - type: nauc_ndcg_at_3_diff1 + value: 49.7927 + - type: nauc_ndcg_at_5_max + value: 47.280899999999995 + - type: nauc_ndcg_at_5_std + value: 14.5061 + - type: nauc_ndcg_at_5_diff1 + value: 48.9168 + - type: nauc_ndcg_at_10_max + value: 47.5137 + - type: nauc_ndcg_at_10_std + value: 15.4698 + - type: nauc_ndcg_at_10_diff1 + value: 48.4279 + - type: nauc_ndcg_at_20_max + value: 47.9904 + - type: nauc_ndcg_at_20_std + value: 15.7135 + - type: nauc_ndcg_at_20_diff1 + value: 48.4332 + - type: nauc_ndcg_at_100_max + value: 48.2942 + - type: nauc_ndcg_at_100_std + value: 17.502100000000002 + - type: nauc_ndcg_at_100_diff1 + value: 48.6035 + - type: nauc_ndcg_at_1000_max + value: 48.0957 + - type: nauc_ndcg_at_1000_std + value: 17.6368 + - type: nauc_ndcg_at_1000_diff1 + value: 48.7597 + - type: nauc_map_at_1_max + value: 45.6445 + - type: nauc_map_at_1_std + value: 6.9397 + - type: nauc_map_at_1_diff1 + value: 58.6992 + - type: nauc_map_at_3_max + value: 45.8449 + - type: nauc_map_at_3_std + value: 11.036200000000001 + - type: nauc_map_at_3_diff1 + value: 51.906 + - type: nauc_map_at_5_max + value: 46.3198 + - type: nauc_map_at_5_std + value: 11.921 + - type: nauc_map_at_5_diff1 + value: 51.2763 + - type: nauc_map_at_10_max + value: 46.5425 + - type: nauc_map_at_10_std + value: 12.5743 + - type: nauc_map_at_10_diff1 + value: 50.9536 + - type: nauc_map_at_20_max + value: 46.726 + - type: nauc_map_at_20_std + value: 12.6497 + - type: nauc_map_at_20_diff1 + value: 50.99510000000001 + - type: nauc_map_at_100_max + value: 46.7746 + - type: nauc_map_at_100_std + value: 12.881200000000002 + - type: nauc_map_at_100_diff1 + value: 51.011399999999995 + - type: nauc_map_at_1000_max + value: 46.785900000000005 + - type: nauc_map_at_1000_std + value: 12.898000000000001 + - type: nauc_map_at_1000_diff1 + value: 51.01480000000001 + - type: nauc_recall_at_1_max + value: 45.6445 + - type: nauc_recall_at_1_std + value: 6.9397 + - type: nauc_recall_at_1_diff1 + value: 58.6992 + - type: nauc_recall_at_3_max + value: 45.0182 + - type: nauc_recall_at_3_std + value: 14.2648 + - type: nauc_recall_at_3_diff1 + value: 45.3428 + - type: nauc_recall_at_5_max + value: 46.2258 + - type: nauc_recall_at_5_std + value: 17.2103 + - type: nauc_recall_at_5_diff1 + value: 42.5614 + - type: nauc_recall_at_10_max + value: 46.251799999999996 + - type: nauc_recall_at_10_std + value: 19.8669 + - type: nauc_recall_at_10_diff1 + value: 40.415 + - type: nauc_recall_at_20_max + value: 46.7318 + - type: nauc_recall_at_20_std + value: 20.3996 + - type: nauc_recall_at_20_diff1 + value: 39.0112 + - type: nauc_recall_at_100_max + value: 48.3756 + - type: nauc_recall_at_100_std + value: 33.558 + - type: nauc_recall_at_100_diff1 + value: 37.584 + - type: nauc_recall_at_1000_max + value: 46.1278 + - type: nauc_recall_at_1000_std + value: 50.2506 + - type: nauc_recall_at_1000_diff1 + value: 33.7694 + - type: nauc_precision_at_1_max + value: 49.1944 + - type: nauc_precision_at_1_std + value: 11.7093 + - type: nauc_precision_at_1_diff1 + value: 56.8806 + - type: nauc_precision_at_3_max + value: 49.9406 + - type: nauc_precision_at_3_std + value: 22.883200000000002 + - type: nauc_precision_at_3_diff1 + value: 40.5974 + - type: nauc_precision_at_5_max + value: 48.4187 + - type: nauc_precision_at_5_std + value: 25.9129 + - type: nauc_precision_at_5_diff1 + value: 34.863 + - type: nauc_precision_at_10_max + value: 46.734700000000004 + - type: nauc_precision_at_10_std + value: 28.5765 + - type: nauc_precision_at_10_diff1 + value: 30.071599999999997 + - type: nauc_precision_at_20_max + value: 45.2343 + - type: nauc_precision_at_20_std + value: 27.4324 + - type: nauc_precision_at_20_diff1 + value: 26.888299999999997 + - type: nauc_precision_at_100_max + value: 33.7511 + - type: nauc_precision_at_100_std + value: 30.084300000000002 + - type: nauc_precision_at_100_diff1 + value: 14.877099999999999 + - type: nauc_precision_at_1000_max + value: 15.059000000000001 + - type: nauc_precision_at_1000_std + value: 21.4471 + - type: nauc_precision_at_1000_diff1 + value: -1.2862 + - type: nauc_mrr_at_1_max + value: 49.1944 + - type: nauc_mrr_at_1_std + value: 11.7093 + - type: nauc_mrr_at_1_diff1 + value: 56.8806 + - type: nauc_mrr_at_3_max + value: 48.8173 + - type: nauc_mrr_at_3_std + value: 14.7023 + - type: nauc_mrr_at_3_diff1 + value: 50.9845 + - type: nauc_mrr_at_5_max + value: 49.0933 + - type: nauc_mrr_at_5_std + value: 15.5443 + - type: nauc_mrr_at_5_diff1 + value: 50.403299999999994 + - type: nauc_mrr_at_10_max + value: 49.058 + - type: nauc_mrr_at_10_std + value: 15.6592 + - type: nauc_mrr_at_10_diff1 + value: 50.3304 + - type: nauc_mrr_at_20_max + value: 49.104 + - type: nauc_mrr_at_20_std + value: 15.7446 + - type: nauc_mrr_at_20_diff1 + value: 50.2689 + - type: nauc_mrr_at_100_max + value: 49.071999999999996 + - type: nauc_mrr_at_100_std + value: 15.8584 + - type: nauc_mrr_at_100_diff1 + value: 50.3045 + - type: nauc_mrr_at_1000_max + value: 49.061 + - type: nauc_mrr_at_1000_std + value: 15.856700000000002 + - type: nauc_mrr_at_1000_diff1 + value: 50.3081 + - type: main_score + value: 36.796 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackTexRetrieval (default) + revision: 46989137a86843e03a6195de44b09deda022eec7 + split: test + type: mteb/cqadupstack-tex + metrics: + - type: ndcg_at_1 + value: 23.159 - type: ndcg_at_3 - value: 0.76031 + value: 27.401999999999997 - type: ndcg_at_5 - value: 0.77915 + value: 29.354000000000003 + - type: ndcg_at_10 + value: 31.775 + - type: ndcg_at_20 + value: 33.743 + - type: ndcg_at_100 + value: 37.125 + - type: ndcg_at_1000 + value: 39.956 + - type: map_at_1 + value: 18.997 + - type: map_at_3 + value: 24.351 + - type: map_at_5 + value: 25.724999999999998 + - type: map_at_10 + value: 26.873 + - type: map_at_20 + value: 27.479 + - type: map_at_100 + value: 28.008 + - type: map_at_1000 + value: 28.133999999999997 - type: recall_at_1 - value: 0.63084 + value: 18.997 + - type: recall_at_3 + value: 30.14 + - type: recall_at_5 + value: 35.225 - type: recall_at_10 - value: 0.92411 + value: 42.447 + - type: recall_at_20 + value: 49.769000000000005 - type: recall_at_100 - value: 0.97363 + value: 66.39500000000001 - type: recall_at_1000 - value: 0.98833 - - type: recall_at_20 - value: 0.9374 + value: 86.434 + - type: precision_at_1 + value: 23.159 + - type: precision_at_3 + value: 12.995999999999999 + - type: precision_at_5 + value: 9.381 + - type: precision_at_10 + value: 5.778 + - type: precision_at_20 + value: 3.467 + - type: precision_at_100 + value: 0.9900000000000001 + - type: precision_at_1000 + value: 0.14200000000000002 + - type: mrr_at_1 + value: 23.159 + - type: mrr_at_3 + value: 28.676299999999998 + - type: mrr_at_5 + value: 29.9082 + - type: mrr_at_10 + value: 30.9286 + - type: mrr_at_20 + value: 31.4303 + - type: mrr_at_100 + value: 31.845000000000002 + - type: mrr_at_1000 + value: 31.9176 + - type: nauc_ndcg_at_1_max + value: 32.959500000000006 + - type: nauc_ndcg_at_1_std + value: -2.0082 + - type: nauc_ndcg_at_1_diff1 + value: 41.801500000000004 + - type: nauc_ndcg_at_3_max + value: 32.8362 + - type: nauc_ndcg_at_3_std + value: -0.9611 + - type: nauc_ndcg_at_3_diff1 + value: 36.248200000000004 + - type: nauc_ndcg_at_5_max + value: 32.650800000000004 + - type: nauc_ndcg_at_5_std + value: 0.13879999999999998 + - type: nauc_ndcg_at_5_diff1 + value: 35.2211 + - type: nauc_ndcg_at_10_max + value: 32.6256 + - type: nauc_ndcg_at_10_std + value: 1.0654000000000001 + - type: nauc_ndcg_at_10_diff1 + value: 34.6558 + - type: nauc_ndcg_at_20_max + value: 33.0706 + - type: nauc_ndcg_at_20_std + value: 2.2485 + - type: nauc_ndcg_at_20_diff1 + value: 34.5314 + - type: nauc_ndcg_at_100_max + value: 33.3131 + - type: nauc_ndcg_at_100_std + value: 3.4467 + - type: nauc_ndcg_at_100_diff1 + value: 34.4791 + - type: nauc_ndcg_at_1000_max + value: 33.644400000000005 + - type: nauc_ndcg_at_1000_std + value: 3.6159999999999997 + - type: nauc_ndcg_at_1000_diff1 + value: 34.9717 + - type: nauc_map_at_1_max + value: 30.2696 + - type: nauc_map_at_1_std + value: -3.3264 + - type: nauc_map_at_1_diff1 + value: 42.0066 + - type: nauc_map_at_3_max + value: 31.455899999999996 + - type: nauc_map_at_3_std + value: -1.8429999999999997 + - type: nauc_map_at_3_diff1 + value: 37.4893 + - type: nauc_map_at_5_max + value: 31.7755 + - type: nauc_map_at_5_std + value: -1.1461999999999999 + - type: nauc_map_at_5_diff1 + value: 36.8624 + - type: nauc_map_at_10_max + value: 31.9842 + - type: nauc_map_at_10_std + value: -0.6542 + - type: nauc_map_at_10_diff1 + value: 36.5911 + - type: nauc_map_at_20_max + value: 32.1745 + - type: nauc_map_at_20_std + value: -0.2191 + - type: nauc_map_at_20_diff1 + value: 36.552800000000005 + - type: nauc_map_at_100_max + value: 32.3001 + - type: nauc_map_at_100_std + value: 0.012199999999999999 + - type: nauc_map_at_100_diff1 + value: 36.5376 + - type: nauc_map_at_1000_max + value: 32.3571 + - type: nauc_map_at_1000_std + value: 0.0557 + - type: nauc_map_at_1000_diff1 + value: 36.5535 + - type: nauc_recall_at_1_max + value: 30.2696 + - type: nauc_recall_at_1_std + value: -3.3264 + - type: nauc_recall_at_1_diff1 + value: 42.0066 + - type: nauc_recall_at_3_max + value: 30.413600000000002 + - type: nauc_recall_at_3_std + value: -0.44530000000000003 + - type: nauc_recall_at_3_diff1 + value: 32.3805 + - type: nauc_recall_at_5_max + value: 30.075499999999998 + - type: nauc_recall_at_5_std + value: 1.8853000000000002 + - type: nauc_recall_at_5_diff1 + value: 29.8885 + - type: nauc_recall_at_10_max + value: 29.7039 + - type: nauc_recall_at_10_std + value: 4.1936 + - type: nauc_recall_at_10_diff1 + value: 27.9912 + - type: nauc_recall_at_20_max + value: 30.538700000000002 + - type: nauc_recall_at_20_std + value: 7.8352 + - type: nauc_recall_at_20_diff1 + value: 26.842 + - type: nauc_recall_at_100_max + value: 30.8116 + - type: nauc_recall_at_100_std + value: 15.1426 + - type: nauc_recall_at_100_diff1 + value: 23.9166 + - type: nauc_recall_at_1000_max + value: 31.9647 + - type: nauc_recall_at_1000_std + value: 26.5754 + - type: nauc_recall_at_1000_diff1 + value: 22.608 + - type: nauc_precision_at_1_max + value: 32.959500000000006 + - type: nauc_precision_at_1_std + value: -2.0082 + - type: nauc_precision_at_1_diff1 + value: 41.801500000000004 + - type: nauc_precision_at_3_max + value: 34.8709 + - type: nauc_precision_at_3_std + value: 1.5288 + - type: nauc_precision_at_3_diff1 + value: 30.6782 + - type: nauc_precision_at_5_max + value: 34.163700000000006 + - type: nauc_precision_at_5_std + value: 4.3446 + - type: nauc_precision_at_5_diff1 + value: 26.2964 + - type: nauc_precision_at_10_max + value: 33.1747 + - type: nauc_precision_at_10_std + value: 7.2109000000000005 + - type: nauc_precision_at_10_diff1 + value: 22.6126 + - type: nauc_precision_at_20_max + value: 32.8185 + - type: nauc_precision_at_20_std + value: 11.296100000000001 + - type: nauc_precision_at_20_diff1 + value: 19.4086 + - type: nauc_precision_at_100_max + value: 30.4363 + - type: nauc_precision_at_100_std + value: 14.23 + - type: nauc_precision_at_100_diff1 + value: 13.1689 + - type: nauc_precision_at_1000_max + value: 24.6263 + - type: nauc_precision_at_1000_std + value: 11.190999999999999 + - type: nauc_precision_at_1000_diff1 + value: 4.5375 + - type: nauc_mrr_at_1_max + value: 32.959500000000006 + - type: nauc_mrr_at_1_std + value: -2.0082 + - type: nauc_mrr_at_1_diff1 + value: 41.801500000000004 + - type: nauc_mrr_at_3_max + value: 33.949400000000004 + - type: nauc_mrr_at_3_std + value: -0.5342 + - type: nauc_mrr_at_3_diff1 + value: 37.3148 + - type: nauc_mrr_at_5_max + value: 33.7685 + - type: nauc_mrr_at_5_std + value: 0.2542 + - type: nauc_mrr_at_5_diff1 + value: 36.5632 + - type: nauc_mrr_at_10_max + value: 33.849000000000004 + - type: nauc_mrr_at_10_std + value: 0.6677 + - type: nauc_mrr_at_10_diff1 + value: 36.4741 + - type: nauc_mrr_at_20_max + value: 33.9586 + - type: nauc_mrr_at_20_std + value: 0.897 + - type: nauc_mrr_at_20_diff1 + value: 36.478899999999996 + - type: nauc_mrr_at_100_max + value: 33.9441 + - type: nauc_mrr_at_100_std + value: 0.9808000000000001 + - type: nauc_mrr_at_100_diff1 + value: 36.5049 + - type: nauc_mrr_at_1000_max + value: 33.9546 + - type: nauc_mrr_at_1000_std + value: 0.9831 + - type: nauc_mrr_at_1000_diff1 + value: 36.5259 + - type: main_score + value: 31.775 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackUnixRetrieval (default) + revision: 6c6430d3a6d36f8d2a829195bc5dc94d7e063e53 + split: test + type: mteb/cqadupstack-unix + metrics: + - type: ndcg_at_1 + value: 34.981 + - type: ndcg_at_3 + value: 40.107 + - type: ndcg_at_5 + value: 42.842999999999996 + - type: ndcg_at_10 + value: 45.275 + - type: ndcg_at_20 + value: 47.455999999999996 + - type: ndcg_at_100 + value: 50.321000000000005 + - type: ndcg_at_1000 + value: 52.406 + - type: map_at_1 + value: 29.504 + - type: map_at_3 + value: 36.622 + - type: map_at_5 + value: 38.541 + - type: map_at_10 + value: 39.675 + - type: map_at_20 + value: 40.409 + - type: map_at_100 + value: 40.914 + - type: map_at_1000 + value: 41.012 + - type: recall_at_1 + value: 29.504 - type: recall_at_3 - value: 0.84159 + value: 43.807 - type: recall_at_5 - value: 0.88627 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (th) - config: th - split: dev + value: 50.77700000000001 + - type: recall_at_10 + value: 57.898 + - type: recall_at_20 + value: 65.59899999999999 + - type: recall_at_100 + value: 78.974 + - type: recall_at_1000 + value: 93.33399999999999 + - type: precision_at_1 + value: 34.981 + - type: precision_at_3 + value: 18.315 + - type: precision_at_5 + value: 13.097 + - type: precision_at_10 + value: 7.631 + - type: precision_at_20 + value: 4.431 + - type: precision_at_100 + value: 1.13 + - type: precision_at_1000 + value: 0.14100000000000001 + - type: mrr_at_1 + value: 34.9813 + - type: mrr_at_3 + value: 41.3557 + - type: mrr_at_5 + value: 42.9602 + - type: mrr_at_10 + value: 43.9816 + - type: mrr_at_20 + value: 44.5 + - type: mrr_at_100 + value: 44.8076 + - type: mrr_at_1000 + value: 44.865 + - type: nauc_ndcg_at_1_max + value: 48.6102 + - type: nauc_ndcg_at_1_std + value: -5.6691 + - type: nauc_ndcg_at_1_diff1 + value: 56.008599999999994 + - type: nauc_ndcg_at_3_max + value: 46.388400000000004 + - type: nauc_ndcg_at_3_std + value: -4.877800000000001 + - type: nauc_ndcg_at_3_diff1 + value: 49.1768 + - type: nauc_ndcg_at_5_max + value: 46.3438 + - type: nauc_ndcg_at_5_std + value: -4.1069 + - type: nauc_ndcg_at_5_diff1 + value: 48.209999999999994 + - type: nauc_ndcg_at_10_max + value: 46.147 + - type: nauc_ndcg_at_10_std + value: -3.7115 + - type: nauc_ndcg_at_10_diff1 + value: 47.9846 + - type: nauc_ndcg_at_20_max + value: 46.2731 + - type: nauc_ndcg_at_20_std + value: -3.5068 + - type: nauc_ndcg_at_20_diff1 + value: 48.1901 + - type: nauc_ndcg_at_100_max + value: 46.886 + - type: nauc_ndcg_at_100_std + value: -1.8507 + - type: nauc_ndcg_at_100_diff1 + value: 49.058 + - type: nauc_ndcg_at_1000_max + value: 46.5984 + - type: nauc_ndcg_at_1000_std + value: -2.1614999999999998 + - type: nauc_ndcg_at_1000_diff1 + value: 49.1318 + - type: nauc_map_at_1_max + value: 45.5569 + - type: nauc_map_at_1_std + value: -7.604900000000001 + - type: nauc_map_at_1_diff1 + value: 56.3936 + - type: nauc_map_at_3_max + value: 46.0028 + - type: nauc_map_at_3_std + value: -6.334 + - type: nauc_map_at_3_diff1 + value: 51.3472 + - type: nauc_map_at_5_max + value: 46.2903 + - type: nauc_map_at_5_std + value: -5.475300000000001 + - type: nauc_map_at_5_diff1 + value: 50.5945 + - type: nauc_map_at_10_max + value: 46.3277 + - type: nauc_map_at_10_std + value: -5.1829 + - type: nauc_map_at_10_diff1 + value: 50.4714 + - type: nauc_map_at_20_max + value: 46.5326 + - type: nauc_map_at_20_std + value: -5.0456 + - type: nauc_map_at_20_diff1 + value: 50.5729 + - type: nauc_map_at_100_max + value: 46.6537 + - type: nauc_map_at_100_std + value: -4.7367 + - type: nauc_map_at_100_diff1 + value: 50.711 + - type: nauc_map_at_1000_max + value: 46.6406 + - type: nauc_map_at_1000_std + value: -4.7269 + - type: nauc_map_at_1000_diff1 + value: 50.6985 + - type: nauc_recall_at_1_max + value: 45.5569 + - type: nauc_recall_at_1_std + value: -7.604900000000001 + - type: nauc_recall_at_1_diff1 + value: 56.3936 + - type: nauc_recall_at_3_max + value: 43.1624 + - type: nauc_recall_at_3_std + value: -5.0664 + - type: nauc_recall_at_3_diff1 + value: 44.016 + - type: nauc_recall_at_5_max + value: 42.893 + - type: nauc_recall_at_5_std + value: -2.0581 + - type: nauc_recall_at_5_diff1 + value: 40.6813 + - type: nauc_recall_at_10_max + value: 41.3464 + - type: nauc_recall_at_10_std + value: -0.9026 + - type: nauc_recall_at_10_diff1 + value: 38.8716 + - type: nauc_recall_at_20_max + value: 40.7766 + - type: nauc_recall_at_20_std + value: -0.4664 + - type: nauc_recall_at_20_diff1 + value: 38.6801 + - type: nauc_recall_at_100_max + value: 43.856 + - type: nauc_recall_at_100_std + value: 12.148200000000001 + - type: nauc_recall_at_100_diff1 + value: 43.189899999999994 + - type: nauc_recall_at_1000_max + value: 36.6555 + - type: nauc_recall_at_1000_std + value: 25.7409 + - type: nauc_recall_at_1000_diff1 + value: 44.9133 + - type: nauc_precision_at_1_max + value: 48.6102 + - type: nauc_precision_at_1_std + value: -5.6691 + - type: nauc_precision_at_1_diff1 + value: 56.008599999999994 + - type: nauc_precision_at_3_max + value: 43.2148 + - type: nauc_precision_at_3_std + value: 0.0292 + - type: nauc_precision_at_3_diff1 + value: 35.75 + - type: nauc_precision_at_5_max + value: 39.8562 + - type: nauc_precision_at_5_std + value: 4.105 + - type: nauc_precision_at_5_diff1 + value: 28.4213 + - type: nauc_precision_at_10_max + value: 34.901199999999996 + - type: nauc_precision_at_10_std + value: 6.4718 + - type: nauc_precision_at_10_diff1 + value: 22.785 + - type: nauc_precision_at_20_max + value: 29.151 + - type: nauc_precision_at_20_std + value: 8.213 + - type: nauc_precision_at_20_diff1 + value: 16.6992 + - type: nauc_precision_at_100_max + value: 17.1377 + - type: nauc_precision_at_100_std + value: 16.1652 + - type: nauc_precision_at_100_diff1 + value: 4.4657 + - type: nauc_precision_at_1000_max + value: -2.6889 + - type: nauc_precision_at_1000_std + value: 11.010499999999999 + - type: nauc_precision_at_1000_diff1 + value: -11.0026 + - type: nauc_mrr_at_1_max + value: 48.6102 + - type: nauc_mrr_at_1_std + value: -5.6691 + - type: nauc_mrr_at_1_diff1 + value: 56.008599999999994 + - type: nauc_mrr_at_3_max + value: 47.6571 + - type: nauc_mrr_at_3_std + value: -4.1072999999999995 + - type: nauc_mrr_at_3_diff1 + value: 50.18470000000001 + - type: nauc_mrr_at_5_max + value: 47.6268 + - type: nauc_mrr_at_5_std + value: -3.6222 + - type: nauc_mrr_at_5_diff1 + value: 49.5854 + - type: nauc_mrr_at_10_max + value: 47.454499999999996 + - type: nauc_mrr_at_10_std + value: -3.4977 + - type: nauc_mrr_at_10_diff1 + value: 49.5833 + - type: nauc_mrr_at_20_max + value: 47.3316 + - type: nauc_mrr_at_20_std + value: -3.5721000000000003 + - type: nauc_mrr_at_20_diff1 + value: 49.6713 + - type: nauc_mrr_at_100_max + value: 47.387299999999996 + - type: nauc_mrr_at_100_std + value: -3.4835 + - type: nauc_mrr_at_100_diff1 + value: 49.8135 + - type: nauc_mrr_at_1000_max + value: 47.4002 + - type: nauc_mrr_at_1000_std + value: -3.4842999999999997 + - type: nauc_mrr_at_1000_diff1 + value: 49.8286 + - type: main_score + value: 45.275 task: type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWebmastersRetrieval (default) + revision: 160c094312a0e1facb97e55eeddb698c0abe3571 + split: test + type: mteb/cqadupstack-webmasters metrics: - type: ndcg_at_1 - value: 0.66712 + value: 32.806000000000004 + - type: ndcg_at_3 + value: 38.775999999999996 + - type: ndcg_at_5 + value: 40.614 - type: ndcg_at_10 - value: 0.73324 + value: 42.957 + - type: ndcg_at_20 + value: 45.202999999999996 - type: ndcg_at_100 - value: 0.76633 + value: 48.941 - type: ndcg_at_1000 - value: 0.77119 - - type: ndcg_at_20 - value: 0.75243 + value: 51.105000000000004 + - type: map_at_1 + value: 27.236 + - type: map_at_3 + value: 34.204 + - type: map_at_5 + value: 35.66 + - type: map_at_10 + value: 36.986000000000004 + - type: map_at_20 + value: 37.827 + - type: map_at_100 + value: 38.602 + - type: map_at_1000 + value: 38.818000000000005 + - type: recall_at_1 + value: 27.236 + - type: recall_at_3 + value: 41.596 + - type: recall_at_5 + value: 46.947 + - type: recall_at_10 + value: 54.129000000000005 + - type: recall_at_20 + value: 62.641000000000005 + - type: recall_at_100 + value: 80.971 + - type: recall_at_1000 + value: 93.98100000000001 + - type: precision_at_1 + value: 32.806000000000004 + - type: precision_at_3 + value: 18.445 + - type: precision_at_5 + value: 13.083 + - type: precision_at_10 + value: 8.142000000000001 + - type: precision_at_20 + value: 5.119 + - type: precision_at_100 + value: 1.599 + - type: precision_at_1000 + value: 0.244 + - type: mrr_at_1 + value: 32.8063 + - type: mrr_at_3 + value: 39.5257 + - type: mrr_at_5 + value: 40.8399 + - type: mrr_at_10 + value: 41.8107 + - type: mrr_at_20 + value: 42.4012 + - type: mrr_at_100 + value: 42.7919 + - type: mrr_at_1000 + value: 42.8261 + - type: nauc_ndcg_at_1_max + value: 49.2838 + - type: nauc_ndcg_at_1_std + value: 8.713799999999999 + - type: nauc_ndcg_at_1_diff1 + value: 48.2777 + - type: nauc_ndcg_at_3_max + value: 44.4031 + - type: nauc_ndcg_at_3_std + value: 11.4725 + - type: nauc_ndcg_at_3_diff1 + value: 41.5639 + - type: nauc_ndcg_at_5_max + value: 44.452999999999996 + - type: nauc_ndcg_at_5_std + value: 11.9373 + - type: nauc_ndcg_at_5_diff1 + value: 41.977199999999996 + - type: nauc_ndcg_at_10_max + value: 44.8695 + - type: nauc_ndcg_at_10_std + value: 13.6193 + - type: nauc_ndcg_at_10_diff1 + value: 41.665 + - type: nauc_ndcg_at_20_max + value: 45.691900000000004 + - type: nauc_ndcg_at_20_std + value: 14.0959 + - type: nauc_ndcg_at_20_diff1 + value: 42.2414 + - type: nauc_ndcg_at_100_max + value: 45.7442 + - type: nauc_ndcg_at_100_std + value: 15.218699999999998 + - type: nauc_ndcg_at_100_diff1 + value: 41.7288 + - type: nauc_ndcg_at_1000_max + value: 46.788000000000004 + - type: nauc_ndcg_at_1000_std + value: 15.409900000000002 + - type: nauc_ndcg_at_1000_diff1 + value: 41.9824 + - type: nauc_map_at_1_max + value: 48.0334 + - type: nauc_map_at_1_std + value: 8.0125 + - type: nauc_map_at_1_diff1 + value: 53.4579 + - type: nauc_map_at_3_max + value: 45.1289 + - type: nauc_map_at_3_std + value: 10.013 + - type: nauc_map_at_3_diff1 + value: 45.51 + - type: nauc_map_at_5_max + value: 45.3494 + - type: nauc_map_at_5_std + value: 10.0348 + - type: nauc_map_at_5_diff1 + value: 45.3972 + - type: nauc_map_at_10_max + value: 45.8378 + - type: nauc_map_at_10_std + value: 11.3299 + - type: nauc_map_at_10_diff1 + value: 44.8933 + - type: nauc_map_at_20_max + value: 46.156000000000006 + - type: nauc_map_at_20_std + value: 11.8154 + - type: nauc_map_at_20_diff1 + value: 44.6615 + - type: nauc_map_at_100_max + value: 46.1188 + - type: nauc_map_at_100_std + value: 12.3635 + - type: nauc_map_at_100_diff1 + value: 44.5946 + - type: nauc_map_at_1000_max + value: 46.1113 + - type: nauc_map_at_1000_std + value: 12.526599999999998 + - type: nauc_map_at_1000_diff1 + value: 44.595400000000005 + - type: nauc_recall_at_1_max + value: 48.0334 + - type: nauc_recall_at_1_std + value: 8.0125 + - type: nauc_recall_at_1_diff1 + value: 53.4579 + - type: nauc_recall_at_3_max + value: 39.3688 + - type: nauc_recall_at_3_std + value: 10.3834 + - type: nauc_recall_at_3_diff1 + value: 37.8084 + - type: nauc_recall_at_5_max + value: 39.3184 + - type: nauc_recall_at_5_std + value: 10.509400000000001 + - type: nauc_recall_at_5_diff1 + value: 36.7191 + - type: nauc_recall_at_10_max + value: 38.785599999999995 + - type: nauc_recall_at_10_std + value: 15.781300000000002 + - type: nauc_recall_at_10_diff1 + value: 34.7564 + - type: nauc_recall_at_20_max + value: 39.6075 + - type: nauc_recall_at_20_std + value: 18.0278 + - type: nauc_recall_at_20_diff1 + value: 35.483399999999996 + - type: nauc_recall_at_100_max + value: 36.1361 + - type: nauc_recall_at_100_std + value: 29.1037 + - type: nauc_recall_at_100_diff1 + value: 26.9486 + - type: nauc_recall_at_1000_max + value: 62.4461 + - type: nauc_recall_at_1000_std + value: 57.465599999999995 + - type: nauc_recall_at_1000_diff1 + value: 29.5554 + - type: nauc_precision_at_1_max + value: 49.2838 + - type: nauc_precision_at_1_std + value: 8.713799999999999 + - type: nauc_precision_at_1_diff1 + value: 48.2777 + - type: nauc_precision_at_3_max + value: 36.4572 + - type: nauc_precision_at_3_std + value: 14.3924 + - type: nauc_precision_at_3_diff1 + value: 22.9406 + - type: nauc_precision_at_5_max + value: 32.5803 + - type: nauc_precision_at_5_std + value: 16.4452 + - type: nauc_precision_at_5_diff1 + value: 18.2745 + - type: nauc_precision_at_10_max + value: 27.3789 + - type: nauc_precision_at_10_std + value: 21.0131 + - type: nauc_precision_at_10_diff1 + value: 6.947399999999999 + - type: nauc_precision_at_20_max + value: 22.8404 + - type: nauc_precision_at_20_std + value: 24.6328 + - type: nauc_precision_at_20_diff1 + value: 0.1601 + - type: nauc_precision_at_100_max + value: 2.6098 + - type: nauc_precision_at_100_std + value: 22.3326 + - type: nauc_precision_at_100_diff1 + value: -10.1755 + - type: nauc_precision_at_1000_max + value: -6.730899999999999 + - type: nauc_precision_at_1000_std + value: 18.262900000000002 + - type: nauc_precision_at_1000_diff1 + value: -16.3364 + - type: nauc_mrr_at_1_max + value: 49.2838 + - type: nauc_mrr_at_1_std + value: 8.713799999999999 + - type: nauc_mrr_at_1_diff1 + value: 48.2777 + - type: nauc_mrr_at_3_max + value: 45.8613 + - type: nauc_mrr_at_3_std + value: 10.4584 + - type: nauc_mrr_at_3_diff1 + value: 42.2388 + - type: nauc_mrr_at_5_max + value: 46.1544 + - type: nauc_mrr_at_5_std + value: 11.1434 + - type: nauc_mrr_at_5_diff1 + value: 42.2252 + - type: nauc_mrr_at_10_max + value: 46.2703 + - type: nauc_mrr_at_10_std + value: 11.7714 + - type: nauc_mrr_at_10_diff1 + value: 42.0821 + - type: nauc_mrr_at_20_max + value: 46.4586 + - type: nauc_mrr_at_20_std + value: 11.9329 + - type: nauc_mrr_at_20_diff1 + value: 42.3199 + - type: nauc_mrr_at_100_max + value: 46.4309 + - type: nauc_mrr_at_100_std + value: 11.9458 + - type: nauc_mrr_at_100_diff1 + value: 42.2902 + - type: nauc_mrr_at_1000_max + value: 46.4392 + - type: nauc_mrr_at_1000_std + value: 11.9269 + - type: nauc_mrr_at_1000_diff1 + value: 42.3078 + - type: main_score + value: 42.957 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CQADupstackWordpressRetrieval (default) + revision: 4ffe81d471b1924886b33c7567bfb200e9eec5c4 + split: test + type: mteb/cqadupstack-wordpress + metrics: + - type: ndcg_at_1 + value: 25.692999999999998 - type: ndcg_at_3 - value: 0.67393 + value: 31.375999999999998 - type: ndcg_at_5 - value: 0.70201 + value: 33.617999999999995 + - type: ndcg_at_10 + value: 36.409000000000006 + - type: ndcg_at_20 + value: 38.5 + - type: ndcg_at_100 + value: 41.614000000000004 + - type: ndcg_at_1000 + value: 44.119 + - type: map_at_1 + value: 23.666 + - type: map_at_3 + value: 29.072 + - type: map_at_5 + value: 30.453999999999997 + - type: map_at_10 + value: 31.673000000000002 + - type: map_at_20 + value: 32.256 + - type: map_at_100 + value: 32.721000000000004 + - type: map_at_1000 + value: 32.82 - type: recall_at_1 - value: 0.47106 + value: 23.666 + - type: recall_at_3 + value: 35.693000000000005 + - type: recall_at_5 + value: 40.937 - type: recall_at_10 - value: 0.84294 + value: 48.979 + - type: recall_at_20 + value: 57.028999999999996 - type: recall_at_100 - value: 0.95949 + value: 72.80799999999999 - type: recall_at_1000 - value: 0.98874 - - type: recall_at_20 - value: 0.90085 + value: 91.546 + - type: precision_at_1 + value: 25.692999999999998 + - type: precision_at_3 + value: 13.123999999999999 + - type: precision_at_5 + value: 9.279 + - type: precision_at_10 + value: 5.712 + - type: precision_at_20 + value: 3.3360000000000003 + - type: precision_at_100 + value: 0.8869999999999999 + - type: precision_at_1000 + value: 0.122 + - type: mrr_at_1 + value: 25.6932 + - type: mrr_at_3 + value: 31.2693 + - type: mrr_at_5 + value: 32.4522 + - type: mrr_at_10 + value: 33.6496 + - type: mrr_at_20 + value: 34.208 + - type: mrr_at_100 + value: 34.6132 + - type: mrr_at_1000 + value: 34.6794 + - type: nauc_ndcg_at_1_max + value: 30.436400000000003 + - type: nauc_ndcg_at_1_std + value: -5.177099999999999 + - type: nauc_ndcg_at_1_diff1 + value: 38.9465 + - type: nauc_ndcg_at_3_max + value: 27.759600000000002 + - type: nauc_ndcg_at_3_std + value: -3.7716 + - type: nauc_ndcg_at_3_diff1 + value: 32.0374 + - type: nauc_ndcg_at_5_max + value: 29.284399999999998 + - type: nauc_ndcg_at_5_std + value: -2.1555999999999997 + - type: nauc_ndcg_at_5_diff1 + value: 31.2735 + - type: nauc_ndcg_at_10_max + value: 27.4811 + - type: nauc_ndcg_at_10_std + value: -2.3712 + - type: nauc_ndcg_at_10_diff1 + value: 30.5165 + - type: nauc_ndcg_at_20_max + value: 28.385899999999996 + - type: nauc_ndcg_at_20_std + value: -0.7358 + - type: nauc_ndcg_at_20_diff1 + value: 30.5901 + - type: nauc_ndcg_at_100_max + value: 29.6634 + - type: nauc_ndcg_at_100_std + value: 0.6082 + - type: nauc_ndcg_at_100_diff1 + value: 30.455 + - type: nauc_ndcg_at_1000_max + value: 29.316 + - type: nauc_ndcg_at_1000_std + value: 0.8039 + - type: nauc_ndcg_at_1000_diff1 + value: 30.406699999999997 + - type: nauc_map_at_1_max + value: 28.618900000000004 + - type: nauc_map_at_1_std + value: -5.8273 + - type: nauc_map_at_1_diff1 + value: 39.6434 + - type: nauc_map_at_3_max + value: 27.3257 + - type: nauc_map_at_3_std + value: -4.8353 + - type: nauc_map_at_3_diff1 + value: 33.9743 + - type: nauc_map_at_5_max + value: 28.5433 + - type: nauc_map_at_5_std + value: -3.7222 + - type: nauc_map_at_5_diff1 + value: 33.360099999999996 + - type: nauc_map_at_10_max + value: 27.972399999999997 + - type: nauc_map_at_10_std + value: -3.565 + - type: nauc_map_at_10_diff1 + value: 32.9863 + - type: nauc_map_at_20_max + value: 28.2615 + - type: nauc_map_at_20_std + value: -3.1113 + - type: nauc_map_at_20_diff1 + value: 32.9793 + - type: nauc_map_at_100_max + value: 28.540300000000002 + - type: nauc_map_at_100_std + value: -2.7937 + - type: nauc_map_at_100_diff1 + value: 32.9581 + - type: nauc_map_at_1000_max + value: 28.5349 + - type: nauc_map_at_1000_std + value: -2.7701 + - type: nauc_map_at_1000_diff1 + value: 32.939299999999996 + - type: nauc_recall_at_1_max + value: 28.618900000000004 + - type: nauc_recall_at_1_std + value: -5.8273 + - type: nauc_recall_at_1_diff1 + value: 39.6434 + - type: nauc_recall_at_3_max + value: 25.120199999999997 + - type: nauc_recall_at_3_std + value: -3.4718 + - type: nauc_recall_at_3_diff1 + value: 27.233200000000004 + - type: nauc_recall_at_5_max + value: 28.6985 + - type: nauc_recall_at_5_std + value: 0.1915 + - type: nauc_recall_at_5_diff1 + value: 25.533299999999997 + - type: nauc_recall_at_10_max + value: 23.3717 + - type: nauc_recall_at_10_std + value: -0.9587999999999999 + - type: nauc_recall_at_10_diff1 + value: 23.8178 + - type: nauc_recall_at_20_max + value: 25.923800000000004 + - type: nauc_recall_at_20_std + value: 5.4661 + - type: nauc_recall_at_20_diff1 + value: 23.4099 + - type: nauc_recall_at_100_max + value: 32.182500000000005 + - type: nauc_recall_at_100_std + value: 14.696200000000001 + - type: nauc_recall_at_100_diff1 + value: 20.6716 + - type: nauc_recall_at_1000_max + value: 31.512400000000003 + - type: nauc_recall_at_1000_std + value: 42.5301 + - type: nauc_recall_at_1000_diff1 + value: 10.7694 + - type: nauc_precision_at_1_max + value: 30.436400000000003 + - type: nauc_precision_at_1_std + value: -5.177099999999999 + - type: nauc_precision_at_1_diff1 + value: 38.9465 + - type: nauc_precision_at_3_max + value: 29.1341 + - type: nauc_precision_at_3_std + value: -0.1582 + - type: nauc_precision_at_3_diff1 + value: 25.872600000000002 + - type: nauc_precision_at_5_max + value: 32.7748 + - type: nauc_precision_at_5_std + value: 4.798100000000001 + - type: nauc_precision_at_5_diff1 + value: 21.712400000000002 + - type: nauc_precision_at_10_max + value: 27.396700000000003 + - type: nauc_precision_at_10_std + value: 6.6187 + - type: nauc_precision_at_10_diff1 + value: 16.292499999999997 + - type: nauc_precision_at_20_max + value: 29.6999 + - type: nauc_precision_at_20_std + value: 12.6113 + - type: nauc_precision_at_20_diff1 + value: 14.616399999999999 + - type: nauc_precision_at_100_max + value: 29.297099999999997 + - type: nauc_precision_at_100_std + value: 20.9722 + - type: nauc_precision_at_100_diff1 + value: 1.6410999999999998 + - type: nauc_precision_at_1000_max + value: 2.7286 + - type: nauc_precision_at_1000_std + value: 14.837200000000001 + - type: nauc_precision_at_1000_diff1 + value: -21.584500000000002 + - type: nauc_mrr_at_1_max + value: 30.436400000000003 + - type: nauc_mrr_at_1_std + value: -5.177099999999999 + - type: nauc_mrr_at_1_diff1 + value: 38.9465 + - type: nauc_mrr_at_3_max + value: 29.766199999999998 + - type: nauc_mrr_at_3_std + value: -3.0375 + - type: nauc_mrr_at_3_diff1 + value: 33.568599999999996 + - type: nauc_mrr_at_5_max + value: 30.4582 + - type: nauc_mrr_at_5_std + value: -2.0233 + - type: nauc_mrr_at_5_diff1 + value: 33.1478 + - type: nauc_mrr_at_10_max + value: 29.3877 + - type: nauc_mrr_at_10_std + value: -2.3752 + - type: nauc_mrr_at_10_diff1 + value: 32.5597 + - type: nauc_mrr_at_20_max + value: 29.631400000000003 + - type: nauc_mrr_at_20_std + value: -1.9325999999999999 + - type: nauc_mrr_at_20_diff1 + value: 32.6145 + - type: nauc_mrr_at_100_max + value: 29.7106 + - type: nauc_mrr_at_100_std + value: -1.8483 + - type: nauc_mrr_at_100_diff1 + value: 32.624900000000004 + - type: nauc_mrr_at_1000_max + value: 29.7099 + - type: nauc_mrr_at_1000_std + value: -1.8341 + - type: nauc_mrr_at_1000_diff1 + value: 32.6251 + - type: main_score + value: 36.409000000000006 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ClimateFEVER (default) + revision: 47f2ac6acb640fc46020b02a5b59fdda04d39380 + split: test + type: mteb/climate-fever + metrics: + - type: ndcg_at_1 + value: 26.971 + - type: ndcg_at_3 + value: 24.196 + - type: ndcg_at_5 + value: 25.811 + - type: ndcg_at_10 + value: 29.494 + - type: ndcg_at_20 + value: 32.013999999999996 + - type: ndcg_at_100 + value: 35.989 + - type: ndcg_at_1000 + value: 39.326 + - type: map_at_1 + value: 12.107 + - type: map_at_3 + value: 17.538 + - type: map_at_5 + value: 19.124 + - type: map_at_10 + value: 20.896 + - type: map_at_20 + value: 21.798000000000002 + - type: map_at_100 + value: 22.567 + - type: map_at_1000 + value: 22.746 + - type: recall_at_1 + value: 12.107 - type: recall_at_3 - value: 0.68456 + value: 22.425 - type: recall_at_5 - value: 0.75915 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (yo) - config: yo - split: dev + value: 27.394000000000002 + - type: recall_at_10 + value: 35.57 + - type: recall_at_20 + value: 42.565 + - type: recall_at_100 + value: 57.708000000000006 + - type: recall_at_1000 + value: 76.673 + - type: precision_at_1 + value: 26.971 + - type: precision_at_3 + value: 18.111 + - type: precision_at_5 + value: 13.694 + - type: precision_at_10 + value: 9.303 + - type: precision_at_20 + value: 5.769 + - type: precision_at_100 + value: 1.6320000000000001 + - type: precision_at_1000 + value: 0.22499999999999998 + - type: mrr_at_1 + value: 26.970699999999997 + - type: mrr_at_3 + value: 36.0478 + - type: mrr_at_5 + value: 37.9598 + - type: mrr_at_10 + value: 39.4286 + - type: mrr_at_20 + value: 39.9242 + - type: mrr_at_100 + value: 40.232600000000005 + - type: mrr_at_1000 + value: 40.2711 + - type: nauc_ndcg_at_1_max + value: 30.1498 + - type: nauc_ndcg_at_1_std + value: 9.795 + - type: nauc_ndcg_at_1_diff1 + value: 28.3202 + - type: nauc_ndcg_at_3_max + value: 36.1507 + - type: nauc_ndcg_at_3_std + value: 16.6918 + - type: nauc_ndcg_at_3_diff1 + value: 25.9179 + - type: nauc_ndcg_at_5_max + value: 38.4314 + - type: nauc_ndcg_at_5_std + value: 19.1236 + - type: nauc_ndcg_at_5_diff1 + value: 25.7315 + - type: nauc_ndcg_at_10_max + value: 39.734 + - type: nauc_ndcg_at_10_std + value: 22.795199999999998 + - type: nauc_ndcg_at_10_diff1 + value: 24.5446 + - type: nauc_ndcg_at_20_max + value: 40.0306 + - type: nauc_ndcg_at_20_std + value: 25.0242 + - type: nauc_ndcg_at_20_diff1 + value: 23.7608 + - type: nauc_ndcg_at_100_max + value: 39.881 + - type: nauc_ndcg_at_100_std + value: 26.8935 + - type: nauc_ndcg_at_100_diff1 + value: 23.366600000000002 + - type: nauc_ndcg_at_1000_max + value: 39.6299 + - type: nauc_ndcg_at_1000_std + value: 27.556000000000004 + - type: nauc_ndcg_at_1000_diff1 + value: 23.4406 + - type: nauc_map_at_1_max + value: 36.033500000000004 + - type: nauc_map_at_1_std + value: 9.3902 + - type: nauc_map_at_1_diff1 + value: 33.3389 + - type: nauc_map_at_3_max + value: 38.2772 + - type: nauc_map_at_3_std + value: 14.862 + - type: nauc_map_at_3_diff1 + value: 29.121799999999997 + - type: nauc_map_at_5_max + value: 38.8901 + - type: nauc_map_at_5_std + value: 16.4551 + - type: nauc_map_at_5_diff1 + value: 28.258499999999998 + - type: nauc_map_at_10_max + value: 39.689099999999996 + - type: nauc_map_at_10_std + value: 19.0082 + - type: nauc_map_at_10_diff1 + value: 27.5292 + - type: nauc_map_at_20_max + value: 39.8114 + - type: nauc_map_at_20_std + value: 20.099700000000002 + - type: nauc_map_at_20_diff1 + value: 27.1249 + - type: nauc_map_at_100_max + value: 39.7759 + - type: nauc_map_at_100_std + value: 20.671400000000002 + - type: nauc_map_at_100_diff1 + value: 26.9515 + - type: nauc_map_at_1000_max + value: 39.7635 + - type: nauc_map_at_1000_std + value: 20.7381 + - type: nauc_map_at_1000_diff1 + value: 26.9318 + - type: nauc_recall_at_1_max + value: 36.033500000000004 + - type: nauc_recall_at_1_std + value: 9.3902 + - type: nauc_recall_at_1_diff1 + value: 33.3389 + - type: nauc_recall_at_3_max + value: 37.040099999999995 + - type: nauc_recall_at_3_std + value: 18.421000000000003 + - type: nauc_recall_at_3_diff1 + value: 23.591 + - type: nauc_recall_at_5_max + value: 38.2483 + - type: nauc_recall_at_5_std + value: 21.9791 + - type: nauc_recall_at_5_diff1 + value: 20.9432 + - type: nauc_recall_at_10_max + value: 38.684400000000004 + - type: nauc_recall_at_10_std + value: 27.528000000000002 + - type: nauc_recall_at_10_diff1 + value: 17.874599999999997 + - type: nauc_recall_at_20_max + value: 37.7408 + - type: nauc_recall_at_20_std + value: 31.178800000000003 + - type: nauc_recall_at_20_diff1 + value: 15.3021 + - type: nauc_recall_at_100_max + value: 35.0668 + - type: nauc_recall_at_100_std + value: 35.8934 + - type: nauc_recall_at_100_diff1 + value: 12.0978 + - type: nauc_recall_at_1000_max + value: 33.2113 + - type: nauc_recall_at_1000_std + value: 44.3165 + - type: nauc_recall_at_1000_diff1 + value: 9.6011 + - type: nauc_precision_at_1_max + value: 30.1498 + - type: nauc_precision_at_1_std + value: 9.795 + - type: nauc_precision_at_1_diff1 + value: 28.3202 + - type: nauc_precision_at_3_max + value: 32.1047 + - type: nauc_precision_at_3_std + value: 20.7027 + - type: nauc_precision_at_3_diff1 + value: 18.3366 + - type: nauc_precision_at_5_max + value: 32.9484 + - type: nauc_precision_at_5_std + value: 24.439700000000002 + - type: nauc_precision_at_5_diff1 + value: 16.3709 + - type: nauc_precision_at_10_max + value: 30.626900000000003 + - type: nauc_precision_at_10_std + value: 30.3335 + - type: nauc_precision_at_10_diff1 + value: 10.4378 + - type: nauc_precision_at_20_max + value: 26.875100000000003 + - type: nauc_precision_at_20_std + value: 33.1578 + - type: nauc_precision_at_20_diff1 + value: 6.3161 + - type: nauc_precision_at_100_max + value: 18.5691 + - type: nauc_precision_at_100_std + value: 32.4294 + - type: nauc_precision_at_100_diff1 + value: 1.9001000000000001 + - type: nauc_precision_at_1000_max + value: 5.2522 + - type: nauc_precision_at_1000_std + value: 26.337899999999998 + - type: nauc_precision_at_1000_diff1 + value: -4.2309 + - type: nauc_mrr_at_1_max + value: 30.1498 + - type: nauc_mrr_at_1_std + value: 9.795 + - type: nauc_mrr_at_1_diff1 + value: 28.3202 + - type: nauc_mrr_at_3_max + value: 32.2466 + - type: nauc_mrr_at_3_std + value: 15.6475 + - type: nauc_mrr_at_3_diff1 + value: 24.160899999999998 + - type: nauc_mrr_at_5_max + value: 33.1837 + - type: nauc_mrr_at_5_std + value: 16.8917 + - type: nauc_mrr_at_5_diff1 + value: 24.072499999999998 + - type: nauc_mrr_at_10_max + value: 33.576 + - type: nauc_mrr_at_10_std + value: 17.4501 + - type: nauc_mrr_at_10_diff1 + value: 23.9826 + - type: nauc_mrr_at_20_max + value: 33.5003 + - type: nauc_mrr_at_20_std + value: 17.5104 + - type: nauc_mrr_at_20_diff1 + value: 23.9237 + - type: nauc_mrr_at_100_max + value: 33.455200000000005 + - type: nauc_mrr_at_100_std + value: 17.5181 + - type: nauc_mrr_at_100_diff1 + value: 23.9598 + - type: nauc_mrr_at_1000_max + value: 33.4473 + - type: nauc_mrr_at_1000_std + value: 17.4969 + - type: nauc_mrr_at_1000_diff1 + value: 23.974899999999998 + - type: main_score + value: 29.494 task: type: Retrieval + - dataset: + config: default + name: MTEB CodeFeedbackMT (default) + revision: b0f12fa0c0dd67f59c95a5c33d02aeeb4c398c5f + split: test + type: CoIR-Retrieval/codefeedback-mt metrics: - type: ndcg_at_1 - value: 0.4958 + value: 21.044 + - type: ndcg_at_3 + value: 27.134999999999998 + - type: ndcg_at_5 + value: 29.205 - type: ndcg_at_10 - value: 0.68705 + value: 31.391999999999996 + - type: ndcg_at_20 + value: 33.031 - type: ndcg_at_100 - value: 0.70664 + value: 35.852000000000004 - type: ndcg_at_1000 - value: 0.71197 - - type: ndcg_at_20 - value: 0.698 + value: 38.076 + - type: map_at_1 + value: 21.044 + - type: map_at_3 + value: 25.637 + - type: map_at_5 + value: 26.779999999999998 + - type: map_at_10 + value: 27.683000000000003 + - type: map_at_20 + value: 28.133999999999997 + - type: map_at_100 + value: 28.510999999999996 + - type: map_at_1000 + value: 28.588 + - type: recall_at_1 + value: 21.044 + - type: recall_at_3 + value: 31.468 + - type: recall_at_5 + value: 36.522 + - type: recall_at_10 + value: 43.278 + - type: recall_at_20 + value: 49.748 + - type: recall_at_100 + value: 65.16499999999999 + - type: recall_at_1000 + value: 83.031 + - type: precision_at_1 + value: 21.044 + - type: precision_at_3 + value: 10.488999999999999 + - type: precision_at_5 + value: 7.303999999999999 + - type: precision_at_10 + value: 4.328 + - type: precision_at_20 + value: 2.487 + - type: precision_at_100 + value: 0.652 + - type: precision_at_1000 + value: 0.083 + - type: mrr_at_1 + value: 21.043899999999997 + - type: mrr_at_3 + value: 25.6371 + - type: mrr_at_5 + value: 26.7796 + - type: mrr_at_10 + value: 27.6831 + - type: mrr_at_20 + value: 28.1344 + - type: mrr_at_100 + value: 28.510999999999996 + - type: mrr_at_1000 + value: 28.588400000000004 + - type: nauc_ndcg_at_1_max + value: 11.8658 + - type: nauc_ndcg_at_1_std + value: -18.4852 + - type: nauc_ndcg_at_1_diff1 + value: 47.3429 + - type: nauc_ndcg_at_3_max + value: 11.608400000000001 + - type: nauc_ndcg_at_3_std + value: -19.0804 + - type: nauc_ndcg_at_3_diff1 + value: 41.7031 + - type: nauc_ndcg_at_5_max + value: 11.289299999999999 + - type: nauc_ndcg_at_5_std + value: -19.3124 + - type: nauc_ndcg_at_5_diff1 + value: 40.5381 + - type: nauc_ndcg_at_10_max + value: 11.6701 + - type: nauc_ndcg_at_10_std + value: -18.7838 + - type: nauc_ndcg_at_10_diff1 + value: 39.8088 + - type: nauc_ndcg_at_20_max + value: 11.942400000000001 + - type: nauc_ndcg_at_20_std + value: -18.123900000000003 + - type: nauc_ndcg_at_20_diff1 + value: 38.967800000000004 + - type: nauc_ndcg_at_100_max + value: 13.114999999999998 + - type: nauc_ndcg_at_100_std + value: -16.1964 + - type: nauc_ndcg_at_100_diff1 + value: 39.0077 + - type: nauc_ndcg_at_1000_max + value: 13.5244 + - type: nauc_ndcg_at_1000_std + value: -15.2702 + - type: nauc_ndcg_at_1000_diff1 + value: 39.1235 + - type: nauc_map_at_1_max + value: 11.8658 + - type: nauc_map_at_1_std + value: -18.4852 + - type: nauc_map_at_1_diff1 + value: 47.3429 + - type: nauc_map_at_3_max + value: 11.6937 + - type: nauc_map_at_3_std + value: -18.9625 + - type: nauc_map_at_3_diff1 + value: 42.993900000000004 + - type: nauc_map_at_5_max + value: 11.5064 + - type: nauc_map_at_5_std + value: -19.0958 + - type: nauc_map_at_5_diff1 + value: 42.3108 + - type: nauc_map_at_10_max + value: 11.6615 + - type: nauc_map_at_10_std + value: -18.885199999999998 + - type: nauc_map_at_10_diff1 + value: 41.993399999999994 + - type: nauc_map_at_20_max + value: 11.7419 + - type: nauc_map_at_20_std + value: -18.7005 + - type: nauc_map_at_20_diff1 + value: 41.7643 + - type: nauc_map_at_100_max + value: 11.902600000000001 + - type: nauc_map_at_100_std + value: -18.4376 + - type: nauc_map_at_100_diff1 + value: 41.7771 + - type: nauc_map_at_1000_max + value: 11.9208 + - type: nauc_map_at_1000_std + value: -18.395500000000002 + - type: nauc_map_at_1000_diff1 + value: 41.7802 + - type: nauc_recall_at_1_max + value: 11.8658 + - type: nauc_recall_at_1_std + value: -18.4852 + - type: nauc_recall_at_1_diff1 + value: 47.3429 + - type: nauc_recall_at_3_max + value: 11.3724 + - type: nauc_recall_at_3_std + value: -19.3869 + - type: nauc_recall_at_3_diff1 + value: 38.2763 + - type: nauc_recall_at_5_max + value: 10.678600000000001 + - type: nauc_recall_at_5_std + value: -19.8995 + - type: nauc_recall_at_5_diff1 + value: 35.781400000000005 + - type: nauc_recall_at_10_max + value: 11.7997 + - type: nauc_recall_at_10_std + value: -18.3219 + - type: nauc_recall_at_10_diff1 + value: 33.7507 + - type: nauc_recall_at_20_max + value: 12.7832 + - type: nauc_recall_at_20_std + value: -15.8611 + - type: nauc_recall_at_20_diff1 + value: 30.4676 + - type: nauc_recall_at_100_max + value: 20.0012 + - type: nauc_recall_at_100_std + value: -3.8268000000000004 + - type: nauc_recall_at_100_diff1 + value: 28.8928 + - type: nauc_recall_at_1000_max + value: 30.812099999999997 + - type: nauc_recall_at_1000_std + value: 18.1771 + - type: nauc_recall_at_1000_diff1 + value: 23.3851 + - type: nauc_precision_at_1_max + value: 11.8658 + - type: nauc_precision_at_1_std + value: -18.4852 + - type: nauc_precision_at_1_diff1 + value: 47.3429 + - type: nauc_precision_at_3_max + value: 11.3724 + - type: nauc_precision_at_3_std + value: -19.3869 + - type: nauc_precision_at_3_diff1 + value: 38.2763 + - type: nauc_precision_at_5_max + value: 10.678600000000001 + - type: nauc_precision_at_5_std + value: -19.8995 + - type: nauc_precision_at_5_diff1 + value: 35.781400000000005 + - type: nauc_precision_at_10_max + value: 11.7997 + - type: nauc_precision_at_10_std + value: -18.3219 + - type: nauc_precision_at_10_diff1 + value: 33.7507 + - type: nauc_precision_at_20_max + value: 12.7832 + - type: nauc_precision_at_20_std + value: -15.8611 + - type: nauc_precision_at_20_diff1 + value: 30.4676 + - type: nauc_precision_at_100_max + value: 20.0012 + - type: nauc_precision_at_100_std + value: -3.8268000000000004 + - type: nauc_precision_at_100_diff1 + value: 28.8928 + - type: nauc_precision_at_1000_max + value: 30.812099999999997 + - type: nauc_precision_at_1000_std + value: 18.1771 + - type: nauc_precision_at_1000_diff1 + value: 23.3851 + - type: nauc_mrr_at_1_max + value: 11.8658 + - type: nauc_mrr_at_1_std + value: -18.4852 + - type: nauc_mrr_at_1_diff1 + value: 47.3429 + - type: nauc_mrr_at_3_max + value: 11.6937 + - type: nauc_mrr_at_3_std + value: -18.9625 + - type: nauc_mrr_at_3_diff1 + value: 42.993900000000004 + - type: nauc_mrr_at_5_max + value: 11.5064 + - type: nauc_mrr_at_5_std + value: -19.0958 + - type: nauc_mrr_at_5_diff1 + value: 42.3108 + - type: nauc_mrr_at_10_max + value: 11.6615 + - type: nauc_mrr_at_10_std + value: -18.885199999999998 + - type: nauc_mrr_at_10_diff1 + value: 41.993399999999994 + - type: nauc_mrr_at_20_max + value: 11.7419 + - type: nauc_mrr_at_20_std + value: -18.7005 + - type: nauc_mrr_at_20_diff1 + value: 41.7643 + - type: nauc_mrr_at_100_max + value: 11.902600000000001 + - type: nauc_mrr_at_100_std + value: -18.4376 + - type: nauc_mrr_at_100_diff1 + value: 41.7771 + - type: nauc_mrr_at_1000_max + value: 11.9208 + - type: nauc_mrr_at_1000_std + value: -18.395500000000002 + - type: nauc_mrr_at_1000_diff1 + value: 41.7802 + - type: main_score + value: 31.391999999999996 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CodeFeedbackST (default) + revision: d213819e87aab9010628da8b73ab4eb337c89340 + split: test + type: CoIR-Retrieval/codefeedback-st + metrics: + - type: ndcg_at_1 + value: 51.227000000000004 - type: ndcg_at_3 - value: 0.64793 + value: 62.971999999999994 - type: ndcg_at_5 - value: 0.66709 + value: 65.649 + - type: ndcg_at_10 + value: 67.72200000000001 + - type: ndcg_at_20 + value: 68.919 + - type: ndcg_at_100 + value: 70.15299999999999 + - type: ndcg_at_1000 + value: 70.658 + - type: map_at_1 + value: 51.227000000000004 + - type: map_at_3 + value: 60.114000000000004 + - type: map_at_5 + value: 61.607 + - type: map_at_10 + value: 62.475 + - type: map_at_20 + value: 62.806 + - type: map_at_100 + value: 62.979 + - type: map_at_1000 + value: 62.999 - type: recall_at_1 - value: 0.46289 + value: 51.227000000000004 + - type: recall_at_3 + value: 71.232 + - type: recall_at_5 + value: 77.69800000000001 - type: recall_at_10 - value: 0.85154 + value: 84.041 + - type: recall_at_20 + value: 88.756 - type: recall_at_100 - value: 0.93557 + value: 95.371 - type: recall_at_1000 - value: 0.97479 - - type: recall_at_20 - value: 0.89076 + value: 99.278 + - type: precision_at_1 + value: 51.227000000000004 + - type: precision_at_3 + value: 23.744 + - type: precision_at_5 + value: 15.540000000000001 + - type: precision_at_10 + value: 8.404 + - type: precision_at_20 + value: 4.438000000000001 + - type: precision_at_100 + value: 0.954 + - type: precision_at_1000 + value: 0.099 + - type: mrr_at_1 + value: 51.0062 + - type: mrr_at_3 + value: 60.0023 + - type: mrr_at_5 + value: 61.492999999999995 + - type: mrr_at_10 + value: 62.362899999999996 + - type: mrr_at_20 + value: 62.693200000000004 + - type: mrr_at_100 + value: 62.8664 + - type: mrr_at_1000 + value: 62.8866 + - type: nauc_ndcg_at_1_max + value: 5.5119 + - type: nauc_ndcg_at_1_std + value: -27.434599999999996 + - type: nauc_ndcg_at_1_diff1 + value: 67.3476 + - type: nauc_ndcg_at_3_max + value: 11.8474 + - type: nauc_ndcg_at_3_std + value: -30.5305 + - type: nauc_ndcg_at_3_diff1 + value: 61.4515 + - type: nauc_ndcg_at_5_max + value: 12.692700000000002 + - type: nauc_ndcg_at_5_std + value: -30.938 + - type: nauc_ndcg_at_5_diff1 + value: 61.0505 + - type: nauc_ndcg_at_10_max + value: 12.354800000000001 + - type: nauc_ndcg_at_10_std + value: -30.6409 + - type: nauc_ndcg_at_10_diff1 + value: 61.205600000000004 + - type: nauc_ndcg_at_20_max + value: 11.9146 + - type: nauc_ndcg_at_20_std + value: -30.247 + - type: nauc_ndcg_at_20_diff1 + value: 61.5428 + - type: nauc_ndcg_at_100_max + value: 11.5407 + - type: nauc_ndcg_at_100_std + value: -29.561700000000002 + - type: nauc_ndcg_at_100_diff1 + value: 62.06270000000001 + - type: nauc_ndcg_at_1000_max + value: 11.2459 + - type: nauc_ndcg_at_1000_std + value: -29.5751 + - type: nauc_ndcg_at_1000_diff1 + value: 62.28 + - type: nauc_map_at_1_max + value: 5.5119 + - type: nauc_map_at_1_std + value: -27.434599999999996 + - type: nauc_map_at_1_diff1 + value: 67.3476 + - type: nauc_map_at_3_max + value: 10.1298 + - type: nauc_map_at_3_std + value: -29.674899999999997 + - type: nauc_map_at_3_diff1 + value: 62.982000000000006 + - type: nauc_map_at_5_max + value: 10.5075 + - type: nauc_map_at_5_std + value: -29.858600000000003 + - type: nauc_map_at_5_diff1 + value: 62.829299999999996 + - type: nauc_map_at_10_max + value: 10.3459 + - type: nauc_map_at_10_std + value: -29.7338 + - type: nauc_map_at_10_diff1 + value: 62.917699999999996 + - type: nauc_map_at_20_max + value: 10.2198 + - type: nauc_map_at_20_std + value: -29.6284 + - type: nauc_map_at_20_diff1 + value: 63.01409999999999 + - type: nauc_map_at_100_max + value: 10.1683 + - type: nauc_map_at_100_std + value: -29.5448 + - type: nauc_map_at_100_diff1 + value: 63.0794 + - type: nauc_map_at_1000_max + value: 10.1602 + - type: nauc_map_at_1000_std + value: -29.5412 + - type: nauc_map_at_1000_diff1 + value: 63.0874 + - type: nauc_recall_at_1_max + value: 5.5119 + - type: nauc_recall_at_1_std + value: -27.434599999999996 + - type: nauc_recall_at_1_diff1 + value: 67.3476 + - type: nauc_recall_at_3_max + value: 17.8724 + - type: nauc_recall_at_3_std + value: -33.5404 + - type: nauc_recall_at_3_diff1 + value: 56.1172 + - type: nauc_recall_at_5_max + value: 21.945700000000002 + - type: nauc_recall_at_5_std + value: -35.5124 + - type: nauc_recall_at_5_diff1 + value: 53.6154 + - type: nauc_recall_at_10_max + value: 23.1968 + - type: nauc_recall_at_10_std + value: -35.4292 + - type: nauc_recall_at_10_diff1 + value: 51.998900000000006 + - type: nauc_recall_at_20_max + value: 23.4056 + - type: nauc_recall_at_20_std + value: -33.825300000000006 + - type: nauc_recall_at_20_diff1 + value: 51.544900000000005 + - type: nauc_recall_at_100_max + value: 29.2331 + - type: nauc_recall_at_100_std + value: -20.444499999999998 + - type: nauc_recall_at_100_diff1 + value: 51.8606 + - type: nauc_recall_at_1000_max + value: 47.943000000000005 + - type: nauc_recall_at_1000_std + value: 16.1139 + - type: nauc_recall_at_1000_diff1 + value: 49.2407 + - type: nauc_precision_at_1_max + value: 5.5119 + - type: nauc_precision_at_1_std + value: -27.434599999999996 + - type: nauc_precision_at_1_diff1 + value: 67.3476 + - type: nauc_precision_at_3_max + value: 17.8724 + - type: nauc_precision_at_3_std + value: -33.5404 + - type: nauc_precision_at_3_diff1 + value: 56.1172 + - type: nauc_precision_at_5_max + value: 21.945700000000002 + - type: nauc_precision_at_5_std + value: -35.5124 + - type: nauc_precision_at_5_diff1 + value: 53.6154 + - type: nauc_precision_at_10_max + value: 23.1968 + - type: nauc_precision_at_10_std + value: -35.4292 + - type: nauc_precision_at_10_diff1 + value: 51.998900000000006 + - type: nauc_precision_at_20_max + value: 23.4056 + - type: nauc_precision_at_20_std + value: -33.825300000000006 + - type: nauc_precision_at_20_diff1 + value: 51.544900000000005 + - type: nauc_precision_at_100_max + value: 29.2331 + - type: nauc_precision_at_100_std + value: -20.444499999999998 + - type: nauc_precision_at_100_diff1 + value: 51.8606 + - type: nauc_precision_at_1000_max + value: 47.943000000000005 + - type: nauc_precision_at_1000_std + value: 16.1139 + - type: nauc_precision_at_1000_diff1 + value: 49.2407 + - type: nauc_mrr_at_1_max + value: 4.9502 + - type: nauc_mrr_at_1_std + value: -27.426099999999998 + - type: nauc_mrr_at_1_diff1 + value: 67.8214 + - type: nauc_mrr_at_3_max + value: 9.7423 + - type: nauc_mrr_at_3_std + value: -29.674699999999998 + - type: nauc_mrr_at_3_diff1 + value: 63.24340000000001 + - type: nauc_mrr_at_5_max + value: 10.1129 + - type: nauc_mrr_at_5_std + value: -29.871100000000002 + - type: nauc_mrr_at_5_diff1 + value: 63.1148 + - type: nauc_mrr_at_10_max + value: 9.9493 + - type: nauc_mrr_at_10_std + value: -29.7413 + - type: nauc_mrr_at_10_diff1 + value: 63.2057 + - type: nauc_mrr_at_20_max + value: 9.8157 + - type: nauc_mrr_at_20_std + value: -29.644 + - type: nauc_mrr_at_20_diff1 + value: 63.307100000000005 + - type: nauc_mrr_at_100_max + value: 9.7639 + - type: nauc_mrr_at_100_std + value: -29.5582 + - type: nauc_mrr_at_100_diff1 + value: 63.3738 + - type: nauc_mrr_at_1000_max + value: 9.7555 + - type: nauc_mrr_at_1000_std + value: -29.554599999999997 + - type: nauc_mrr_at_1000_diff1 + value: 63.382000000000005 + - type: main_score + value: 67.72200000000001 + task: + type: Retrieval + - dataset: + config: python + name: MTEB CodeSearchNetCCRetrieval (python) + revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 + split: test + type: CoIR-Retrieval/CodeSearchNet-ccr + metrics: + - type: ndcg_at_1 + value: 32.417 + - type: ndcg_at_3 + value: 40.904 + - type: ndcg_at_5 + value: 43.321 + - type: ndcg_at_10 + value: 45.532000000000004 + - type: ndcg_at_20 + value: 47.071000000000005 + - type: ndcg_at_100 + value: 49.297999999999995 + - type: ndcg_at_1000 + value: 50.859 + - type: map_at_1 + value: 32.417 + - type: map_at_3 + value: 38.829 + - type: map_at_5 + value: 40.166000000000004 + - type: map_at_10 + value: 41.087 + - type: map_at_20 + value: 41.510999999999996 + - type: map_at_100 + value: 41.815000000000005 + - type: map_at_1000 + value: 41.869 + - type: recall_at_1 + value: 32.417 - type: recall_at_3 - value: 0.7507 + value: 46.903 - type: recall_at_5 - value: 0.79202 - - dataset: - type: miracl/mmteb-miracl - name: Miracl (zh) - config: zh - split: dev + value: 52.788999999999994 + - type: recall_at_10 + value: 59.57900000000001 + - type: recall_at_20 + value: 65.652 + - type: recall_at_100 + value: 77.718 + - type: recall_at_1000 + value: 90.294 + - type: precision_at_1 + value: 32.417 + - type: precision_at_3 + value: 15.634 + - type: precision_at_5 + value: 10.558 + - type: precision_at_10 + value: 5.958 + - type: precision_at_20 + value: 3.283 + - type: precision_at_100 + value: 0.777 + - type: precision_at_1000 + value: 0.09 + - type: mrr_at_1 + value: 32.4239 + - type: mrr_at_3 + value: 38.8323 + - type: mrr_at_5 + value: 40.1696 + - type: mrr_at_10 + value: 41.0908 + - type: mrr_at_20 + value: 41.5149 + - type: mrr_at_100 + value: 41.8188 + - type: mrr_at_1000 + value: 41.8726 + - type: nauc_ndcg_at_1_max + value: 32.4803 + - type: nauc_ndcg_at_1_std + value: -1.1774 + - type: nauc_ndcg_at_1_diff1 + value: 54.68730000000001 + - type: nauc_ndcg_at_3_max + value: 33.5662 + - type: nauc_ndcg_at_3_std + value: 0.361 + - type: nauc_ndcg_at_3_diff1 + value: 49.522 + - type: nauc_ndcg_at_5_max + value: 33.0861 + - type: nauc_ndcg_at_5_std + value: 0.5551999999999999 + - type: nauc_ndcg_at_5_diff1 + value: 48.9052 + - type: nauc_ndcg_at_10_max + value: 33.0427 + - type: nauc_ndcg_at_10_std + value: 1.466 + - type: nauc_ndcg_at_10_diff1 + value: 48.3256 + - type: nauc_ndcg_at_20_max + value: 33.059 + - type: nauc_ndcg_at_20_std + value: 2.2277 + - type: nauc_ndcg_at_20_diff1 + value: 48.2916 + - type: nauc_ndcg_at_100_max + value: 33.0797 + - type: nauc_ndcg_at_100_std + value: 2.9991 + - type: nauc_ndcg_at_100_diff1 + value: 48.266999999999996 + - type: nauc_ndcg_at_1000_max + value: 33.1052 + - type: nauc_ndcg_at_1000_std + value: 2.8583000000000003 + - type: nauc_ndcg_at_1000_diff1 + value: 48.5209 + - type: nauc_map_at_1_max + value: 32.4803 + - type: nauc_map_at_1_std + value: -1.1774 + - type: nauc_map_at_1_diff1 + value: 54.68730000000001 + - type: nauc_map_at_3_max + value: 33.3014 + - type: nauc_map_at_3_std + value: -0.06409999999999999 + - type: nauc_map_at_3_diff1 + value: 50.6726 + - type: nauc_map_at_5_max + value: 33.0327 + - type: nauc_map_at_5_std + value: 0.0325 + - type: nauc_map_at_5_diff1 + value: 50.3363 + - type: nauc_map_at_10_max + value: 33.0181 + - type: nauc_map_at_10_std + value: 0.3939 + - type: nauc_map_at_10_diff1 + value: 50.1109 + - type: nauc_map_at_20_max + value: 33.0183 + - type: nauc_map_at_20_std + value: 0.5951 + - type: nauc_map_at_20_diff1 + value: 50.108 + - type: nauc_map_at_100_max + value: 33.022 + - type: nauc_map_at_100_std + value: 0.6973 + - type: nauc_map_at_100_diff1 + value: 50.10790000000001 + - type: nauc_map_at_1000_max + value: 33.022 + - type: nauc_map_at_1000_std + value: 0.6931999999999999 + - type: nauc_map_at_1000_diff1 + value: 50.1174 + - type: nauc_recall_at_1_max + value: 32.4803 + - type: nauc_recall_at_1_std + value: -1.1774 + - type: nauc_recall_at_1_diff1 + value: 54.68730000000001 + - type: nauc_recall_at_3_max + value: 34.3301 + - type: nauc_recall_at_3_std + value: 1.6075 + - type: nauc_recall_at_3_diff1 + value: 46.2477 + - type: nauc_recall_at_5_max + value: 33.177299999999995 + - type: nauc_recall_at_5_std + value: 2.1687000000000003 + - type: nauc_recall_at_5_diff1 + value: 44.61 + - type: nauc_recall_at_10_max + value: 33.020500000000006 + - type: nauc_recall_at_10_std + value: 5.3331 + - type: nauc_recall_at_10_diff1 + value: 42.3796 + - type: nauc_recall_at_20_max + value: 33.1279 + - type: nauc_recall_at_20_std + value: 9.2437 + - type: nauc_recall_at_20_diff1 + value: 41.584199999999996 + - type: nauc_recall_at_100_max + value: 33.2882 + - type: nauc_recall_at_100_std + value: 18.1866 + - type: nauc_recall_at_100_diff1 + value: 38.9221 + - type: nauc_recall_at_1000_max + value: 34.2607 + - type: nauc_recall_at_1000_std + value: 30.5699 + - type: nauc_recall_at_1000_diff1 + value: 35.204800000000006 + - type: nauc_precision_at_1_max + value: 32.4803 + - type: nauc_precision_at_1_std + value: -1.1774 + - type: nauc_precision_at_1_diff1 + value: 54.68730000000001 + - type: nauc_precision_at_3_max + value: 34.3301 + - type: nauc_precision_at_3_std + value: 1.6075 + - type: nauc_precision_at_3_diff1 + value: 46.2477 + - type: nauc_precision_at_5_max + value: 33.177299999999995 + - type: nauc_precision_at_5_std + value: 2.1687000000000003 + - type: nauc_precision_at_5_diff1 + value: 44.61 + - type: nauc_precision_at_10_max + value: 33.020500000000006 + - type: nauc_precision_at_10_std + value: 5.3331 + - type: nauc_precision_at_10_diff1 + value: 42.3796 + - type: nauc_precision_at_20_max + value: 33.1279 + - type: nauc_precision_at_20_std + value: 9.2437 + - type: nauc_precision_at_20_diff1 + value: 41.584199999999996 + - type: nauc_precision_at_100_max + value: 33.2882 + - type: nauc_precision_at_100_std + value: 18.1866 + - type: nauc_precision_at_100_diff1 + value: 38.9221 + - type: nauc_precision_at_1000_max + value: 34.2607 + - type: nauc_precision_at_1000_std + value: 30.5699 + - type: nauc_precision_at_1000_diff1 + value: 35.204800000000006 + - type: nauc_mrr_at_1_max + value: 32.5013 + - type: nauc_mrr_at_1_std + value: -1.1843 + - type: nauc_mrr_at_1_diff1 + value: 54.6663 + - type: nauc_mrr_at_3_max + value: 33.315 + - type: nauc_mrr_at_3_std + value: -0.06849999999999999 + - type: nauc_mrr_at_3_diff1 + value: 50.66460000000001 + - type: nauc_mrr_at_5_max + value: 33.0452 + - type: nauc_mrr_at_5_std + value: 0.0305 + - type: nauc_mrr_at_5_diff1 + value: 50.326499999999996 + - type: nauc_mrr_at_10_max + value: 33.0308 + - type: nauc_mrr_at_10_std + value: 0.39189999999999997 + - type: nauc_mrr_at_10_diff1 + value: 50.101 + - type: nauc_mrr_at_20_max + value: 33.031 + - type: nauc_mrr_at_20_std + value: 0.5930000000000001 + - type: nauc_mrr_at_20_diff1 + value: 50.0981 + - type: nauc_mrr_at_100_max + value: 33.0348 + - type: nauc_mrr_at_100_std + value: 0.6952 + - type: nauc_mrr_at_100_diff1 + value: 50.097899999999996 + - type: nauc_mrr_at_1000_max + value: 33.0348 + - type: nauc_mrr_at_1000_std + value: 0.6910999999999999 + - type: nauc_mrr_at_1000_diff1 + value: 50.1074 + - type: main_score + value: 45.532000000000004 task: type: Retrieval + - dataset: + config: javascript + name: MTEB CodeSearchNetCCRetrieval (javascript) + revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 + split: test + type: CoIR-Retrieval/CodeSearchNet-ccr metrics: - type: ndcg_at_1 - value: 0.47583 + value: 33.364 + - type: ndcg_at_3 + value: 41.943999999999996 + - type: ndcg_at_5 + value: 44.167 - type: ndcg_at_10 - value: 0.52553 + value: 46.024 + - type: ndcg_at_20 + value: 47.508 - type: ndcg_at_100 - value: 0.6 + value: 49.668 - type: ndcg_at_1000 - value: 0.61415 + value: 51.336999999999996 + - type: map_at_1 + value: 33.364 + - type: map_at_3 + value: 39.846 + - type: map_at_5 + value: 41.083999999999996 + - type: map_at_10 + value: 41.85 + - type: map_at_20 + value: 42.254000000000005 + - type: map_at_100 + value: 42.547000000000004 + - type: map_at_1000 + value: 42.601 + - type: recall_at_1 + value: 33.364 + - type: recall_at_3 + value: 48.010000000000005 + - type: recall_at_5 + value: 53.388000000000005 + - type: recall_at_10 + value: 59.131 + - type: recall_at_20 + value: 65.026 + - type: recall_at_100 + value: 76.755 + - type: recall_at_1000 + value: 90.398 + - type: precision_at_1 + value: 33.364 + - type: precision_at_3 + value: 16.003 + - type: precision_at_5 + value: 10.678 + - type: precision_at_10 + value: 5.913 + - type: precision_at_20 + value: 3.251 + - type: precision_at_100 + value: 0.768 + - type: precision_at_1000 + value: 0.09 + - type: mrr_at_1 + value: 33.272600000000004 + - type: mrr_at_3 + value: 39.7954 + - type: mrr_at_5 + value: 41.0412 + - type: mrr_at_10 + value: 41.8073 + - type: mrr_at_20 + value: 42.2109 + - type: mrr_at_100 + value: 42.5037 + - type: mrr_at_1000 + value: 42.5577 + - type: nauc_ndcg_at_1_max + value: 26.6036 + - type: nauc_ndcg_at_1_std + value: -8.3972 + - type: nauc_ndcg_at_1_diff1 + value: 52.43560000000001 + - type: nauc_ndcg_at_3_max + value: 28.5119 + - type: nauc_ndcg_at_3_std + value: -5.6812000000000005 + - type: nauc_ndcg_at_3_diff1 + value: 47.1671 + - type: nauc_ndcg_at_5_max + value: 28.1875 + - type: nauc_ndcg_at_5_std + value: -5.6434999999999995 + - type: nauc_ndcg_at_5_diff1 + value: 46.1849 + - type: nauc_ndcg_at_10_max + value: 27.5534 + - type: nauc_ndcg_at_10_std + value: -5.6785000000000005 + - type: nauc_ndcg_at_10_diff1 + value: 45.6927 + - type: nauc_ndcg_at_20_max + value: 27.4338 + - type: nauc_ndcg_at_20_std + value: -5.5037 + - type: nauc_ndcg_at_20_diff1 + value: 45.872800000000005 + - type: nauc_ndcg_at_100_max + value: 27.386100000000003 + - type: nauc_ndcg_at_100_std + value: -5.2795000000000005 + - type: nauc_ndcg_at_100_diff1 + value: 46.1008 + - type: nauc_ndcg_at_1000_max + value: 27.5195 + - type: nauc_ndcg_at_1000_std + value: -5.0668999999999995 + - type: nauc_ndcg_at_1000_diff1 + value: 46.381499999999996 + - type: nauc_map_at_1_max + value: 26.6036 + - type: nauc_map_at_1_std + value: -8.3972 + - type: nauc_map_at_1_diff1 + value: 52.43560000000001 + - type: nauc_map_at_3_max + value: 28.098699999999997 + - type: nauc_map_at_3_std + value: -6.357500000000001 + - type: nauc_map_at_3_diff1 + value: 48.4799 + - type: nauc_map_at_5_max + value: 27.938000000000002 + - type: nauc_map_at_5_std + value: -6.3283000000000005 + - type: nauc_map_at_5_diff1 + value: 47.955799999999996 + - type: nauc_map_at_10_max + value: 27.6989 + - type: nauc_map_at_10_std + value: -6.3546000000000005 + - type: nauc_map_at_10_diff1 + value: 47.7813 + - type: nauc_map_at_20_max + value: 27.637099999999997 + - type: nauc_map_at_20_std + value: -6.3278 + - type: nauc_map_at_20_diff1 + value: 47.8258 + - type: nauc_map_at_100_max + value: 27.6654 + - type: nauc_map_at_100_std + value: -6.284199999999999 + - type: nauc_map_at_100_diff1 + value: 47.8675 + - type: nauc_map_at_1000_max + value: 27.668599999999998 + - type: nauc_map_at_1000_std + value: -6.2727 + - type: nauc_map_at_1000_diff1 + value: 47.8793 + - type: nauc_recall_at_1_max + value: 26.6036 + - type: nauc_recall_at_1_std + value: -8.3972 + - type: nauc_recall_at_1_diff1 + value: 52.43560000000001 + - type: nauc_recall_at_3_max + value: 29.686600000000002 + - type: nauc_recall_at_3_std + value: -3.7178999999999998 + - type: nauc_recall_at_3_diff1 + value: 43.3556 + - type: nauc_recall_at_5_max + value: 28.835499999999996 + - type: nauc_recall_at_5_std + value: -3.6023 + - type: nauc_recall_at_5_diff1 + value: 40.7246 + - type: nauc_recall_at_10_max + value: 26.6593 + - type: nauc_recall_at_10_std + value: -3.5498000000000003 + - type: nauc_recall_at_10_diff1 + value: 38.6728 + - type: nauc_recall_at_20_max + value: 26.293499999999998 + - type: nauc_recall_at_20_std + value: -2.3813 + - type: nauc_recall_at_20_diff1 + value: 38.8857 + - type: nauc_recall_at_100_max + value: 24.7411 + - type: nauc_recall_at_100_std + value: 0.1296 + - type: nauc_recall_at_100_diff1 + value: 38.1683 + - type: nauc_recall_at_1000_max + value: 25.1934 + - type: nauc_recall_at_1000_std + value: 10.7766 + - type: nauc_recall_at_1000_diff1 + value: 35.856300000000005 + - type: nauc_precision_at_1_max + value: 26.6036 + - type: nauc_precision_at_1_std + value: -8.3972 + - type: nauc_precision_at_1_diff1 + value: 52.43560000000001 + - type: nauc_precision_at_3_max + value: 29.686600000000002 + - type: nauc_precision_at_3_std + value: -3.7178999999999998 + - type: nauc_precision_at_3_diff1 + value: 43.3556 + - type: nauc_precision_at_5_max + value: 28.835499999999996 + - type: nauc_precision_at_5_std + value: -3.6023 + - type: nauc_precision_at_5_diff1 + value: 40.7246 + - type: nauc_precision_at_10_max + value: 26.6593 + - type: nauc_precision_at_10_std + value: -3.5498000000000003 + - type: nauc_precision_at_10_diff1 + value: 38.6728 + - type: nauc_precision_at_20_max + value: 26.293499999999998 + - type: nauc_precision_at_20_std + value: -2.3813 + - type: nauc_precision_at_20_diff1 + value: 38.8857 + - type: nauc_precision_at_100_max + value: 24.7411 + - type: nauc_precision_at_100_std + value: 0.1296 + - type: nauc_precision_at_100_diff1 + value: 38.1683 + - type: nauc_precision_at_1000_max + value: 25.1934 + - type: nauc_precision_at_1000_std + value: 10.7766 + - type: nauc_precision_at_1000_diff1 + value: 35.856300000000005 + - type: nauc_mrr_at_1_max + value: 26.7351 + - type: nauc_mrr_at_1_std + value: -8.2798 + - type: nauc_mrr_at_1_diff1 + value: 52.7186 + - type: nauc_mrr_at_3_max + value: 28.1671 + - type: nauc_mrr_at_3_std + value: -6.3235 + - type: nauc_mrr_at_3_diff1 + value: 48.6387 + - type: nauc_mrr_at_5_max + value: 28.0115 + - type: nauc_mrr_at_5_std + value: -6.256399999999999 + - type: nauc_mrr_at_5_diff1 + value: 48.098400000000005 + - type: nauc_mrr_at_10_max + value: 27.7729 + - type: nauc_mrr_at_10_std + value: -6.2821 + - type: nauc_mrr_at_10_diff1 + value: 47.925000000000004 + - type: nauc_mrr_at_20_max + value: 27.7115 + - type: nauc_mrr_at_20_std + value: -6.254899999999999 + - type: nauc_mrr_at_20_diff1 + value: 47.9703 + - type: nauc_mrr_at_100_max + value: 27.740199999999998 + - type: nauc_mrr_at_100_std + value: -6.2109 + - type: nauc_mrr_at_100_diff1 + value: 48.0128 + - type: nauc_mrr_at_1000_max + value: 27.743499999999997 + - type: nauc_mrr_at_1000_std + value: -6.1993 + - type: nauc_mrr_at_1000_diff1 + value: 48.0248 + - type: main_score + value: 46.024 + task: + type: Retrieval + - dataset: + config: go + name: MTEB CodeSearchNetCCRetrieval (go) + revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 + split: test + type: CoIR-Retrieval/CodeSearchNet-ccr + metrics: + - type: ndcg_at_1 + value: 26.471 + - type: ndcg_at_3 + value: 33.489999999999995 + - type: ndcg_at_5 + value: 35.55 + - type: ndcg_at_10 + value: 37.555 - type: ndcg_at_20 - value: 0.55668 + value: 39.029 + - type: ndcg_at_100 + value: 41.478 + - type: ndcg_at_1000 + value: 43.457 + - type: map_at_1 + value: 26.471 + - type: map_at_3 + value: 31.774 + - type: map_at_5 + value: 32.915 + - type: map_at_10 + value: 33.745999999999995 + - type: map_at_20 + value: 34.150000000000006 + - type: map_at_100 + value: 34.477999999999994 + - type: map_at_1000 + value: 34.544000000000004 + - type: recall_at_1 + value: 26.471 + - type: recall_at_3 + value: 38.451 + - type: recall_at_5 + value: 43.462 + - type: recall_at_10 + value: 49.643 + - type: recall_at_20 + value: 55.479 + - type: recall_at_100 + value: 68.825 + - type: recall_at_1000 + value: 84.93 + - type: precision_at_1 + value: 26.471 + - type: precision_at_3 + value: 12.817 + - type: precision_at_5 + value: 8.692 + - type: precision_at_10 + value: 4.9639999999999995 + - type: precision_at_20 + value: 2.774 + - type: precision_at_100 + value: 0.688 + - type: precision_at_1000 + value: 0.08499999999999999 + - type: mrr_at_1 + value: 26.459 + - type: mrr_at_3 + value: 31.757400000000004 + - type: mrr_at_5 + value: 32.9092 + - type: mrr_at_10 + value: 33.7387 + - type: mrr_at_20 + value: 34.1428 + - type: mrr_at_100 + value: 34.471000000000004 + - type: mrr_at_1000 + value: 34.5364 + - type: nauc_ndcg_at_1_max + value: 29.408299999999997 + - type: nauc_ndcg_at_1_std + value: 1.5685 + - type: nauc_ndcg_at_1_diff1 + value: 45.834599999999995 + - type: nauc_ndcg_at_3_max + value: 27.7526 + - type: nauc_ndcg_at_3_std + value: -0.43810000000000004 + - type: nauc_ndcg_at_3_diff1 + value: 41.272999999999996 + - type: nauc_ndcg_at_5_max + value: 27.2864 + - type: nauc_ndcg_at_5_std + value: -0.37820000000000004 + - type: nauc_ndcg_at_5_diff1 + value: 40.4934 + - type: nauc_ndcg_at_10_max + value: 26.845599999999997 + - type: nauc_ndcg_at_10_std + value: -0.3317 + - type: nauc_ndcg_at_10_diff1 + value: 39.9305 + - type: nauc_ndcg_at_20_max + value: 26.4669 + - type: nauc_ndcg_at_20_std + value: 0.1423 + - type: nauc_ndcg_at_20_diff1 + value: 39.432 + - type: nauc_ndcg_at_100_max + value: 26.3318 + - type: nauc_ndcg_at_100_std + value: 0.8049000000000001 + - type: nauc_ndcg_at_100_diff1 + value: 39.0276 + - type: nauc_ndcg_at_1000_max + value: 26.5895 + - type: nauc_ndcg_at_1000_std + value: 1.0204 + - type: nauc_ndcg_at_1000_diff1 + value: 39.2747 + - type: nauc_map_at_1_max + value: 29.408299999999997 + - type: nauc_map_at_1_std + value: 1.5685 + - type: nauc_map_at_1_diff1 + value: 45.834599999999995 + - type: nauc_map_at_3_max + value: 28.1245 + - type: nauc_map_at_3_std + value: -0.006999999999999999 + - type: nauc_map_at_3_diff1 + value: 42.2701 + - type: nauc_map_at_5_max + value: 27.8563 + - type: nauc_map_at_5_std + value: 0.0204 + - type: nauc_map_at_5_diff1 + value: 41.8294 + - type: nauc_map_at_10_max + value: 27.6709 + - type: nauc_map_at_10_std + value: 0.0262 + - type: nauc_map_at_10_diff1 + value: 41.5973 + - type: nauc_map_at_20_max + value: 27.572000000000003 + - type: nauc_map_at_20_std + value: 0.1652 + - type: nauc_map_at_20_diff1 + value: 41.4683 + - type: nauc_map_at_100_max + value: 27.5573 + - type: nauc_map_at_100_std + value: 0.243 + - type: nauc_map_at_100_diff1 + value: 41.4201 + - type: nauc_map_at_1000_max + value: 27.5663 + - type: nauc_map_at_1000_std + value: 0.254 + - type: nauc_map_at_1000_diff1 + value: 41.4275 + - type: nauc_recall_at_1_max + value: 29.408299999999997 + - type: nauc_recall_at_1_std + value: 1.5685 + - type: nauc_recall_at_1_diff1 + value: 45.834599999999995 + - type: nauc_recall_at_3_max + value: 26.737499999999997 + - type: nauc_recall_at_3_std + value: -1.6067999999999998 + - type: nauc_recall_at_3_diff1 + value: 38.5628 + - type: nauc_recall_at_5_max + value: 25.6664 + - type: nauc_recall_at_5_std + value: -1.4459 + - type: nauc_recall_at_5_diff1 + value: 36.7369 + - type: nauc_recall_at_10_max + value: 24.3156 + - type: nauc_recall_at_10_std + value: -1.25 + - type: nauc_recall_at_10_diff1 + value: 34.959 + - type: nauc_recall_at_20_max + value: 22.6187 + - type: nauc_recall_at_20_std + value: 0.5539 + - type: nauc_recall_at_20_diff1 + value: 32.634299999999996 + - type: nauc_recall_at_100_max + value: 20.8069 + - type: nauc_recall_at_100_std + value: 5.2502 + - type: nauc_recall_at_100_diff1 + value: 28.3304 + - type: nauc_recall_at_1000_max + value: 20.8473 + - type: nauc_recall_at_1000_std + value: 12.2405 + - type: nauc_recall_at_1000_diff1 + value: 24.2366 + - type: nauc_precision_at_1_max + value: 29.408299999999997 + - type: nauc_precision_at_1_std + value: 1.5685 + - type: nauc_precision_at_1_diff1 + value: 45.834599999999995 + - type: nauc_precision_at_3_max + value: 26.737499999999997 + - type: nauc_precision_at_3_std + value: -1.6067999999999998 + - type: nauc_precision_at_3_diff1 + value: 38.5628 + - type: nauc_precision_at_5_max + value: 25.6664 + - type: nauc_precision_at_5_std + value: -1.4459 + - type: nauc_precision_at_5_diff1 + value: 36.7369 + - type: nauc_precision_at_10_max + value: 24.3156 + - type: nauc_precision_at_10_std + value: -1.25 + - type: nauc_precision_at_10_diff1 + value: 34.959 + - type: nauc_precision_at_20_max + value: 22.6187 + - type: nauc_precision_at_20_std + value: 0.5539 + - type: nauc_precision_at_20_diff1 + value: 32.634299999999996 + - type: nauc_precision_at_100_max + value: 20.8069 + - type: nauc_precision_at_100_std + value: 5.2502 + - type: nauc_precision_at_100_diff1 + value: 28.3304 + - type: nauc_precision_at_1000_max + value: 20.8473 + - type: nauc_precision_at_1000_std + value: 12.2405 + - type: nauc_precision_at_1000_diff1 + value: 24.2366 + - type: nauc_mrr_at_1_max + value: 29.435499999999998 + - type: nauc_mrr_at_1_std + value: 1.5623 + - type: nauc_mrr_at_1_diff1 + value: 45.8822 + - type: nauc_mrr_at_3_max + value: 28.183000000000003 + - type: nauc_mrr_at_3_std + value: -0.00039999999999999996 + - type: nauc_mrr_at_3_diff1 + value: 42.2776 + - type: nauc_mrr_at_5_max + value: 27.8735 + - type: nauc_mrr_at_5_std + value: 0.0288 + - type: nauc_mrr_at_5_diff1 + value: 41.827999999999996 + - type: nauc_mrr_at_10_max + value: 27.6989 + - type: nauc_mrr_at_10_std + value: 0.0349 + - type: nauc_mrr_at_10_diff1 + value: 41.6043 + - type: nauc_mrr_at_20_max + value: 27.599 + - type: nauc_mrr_at_20_std + value: 0.1719 + - type: nauc_mrr_at_20_diff1 + value: 41.4786 + - type: nauc_mrr_at_100_max + value: 27.5846 + - type: nauc_mrr_at_100_std + value: 0.25 + - type: nauc_mrr_at_100_diff1 + value: 41.4307 + - type: nauc_mrr_at_1000_max + value: 27.5937 + - type: nauc_mrr_at_1000_std + value: 0.261 + - type: nauc_mrr_at_1000_diff1 + value: 41.4381 + - type: main_score + value: 37.555 + task: + type: Retrieval + - dataset: + config: ruby + name: MTEB CodeSearchNetCCRetrieval (ruby) + revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 + split: test + type: CoIR-Retrieval/CodeSearchNet-ccr + metrics: + - type: ndcg_at_1 + value: 36.003 - type: ndcg_at_3 - value: 0.45839 + value: 43.306 - type: ndcg_at_5 - value: 0.48127 + value: 45.443 + - type: ndcg_at_10 + value: 47.549 + - type: ndcg_at_20 + value: 48.872 + - type: ndcg_at_100 + value: 50.651 + - type: ndcg_at_1000 + value: 52.406 + - type: map_at_1 + value: 36.003 + - type: map_at_3 + value: 41.501 + - type: map_at_5 + value: 42.695 + - type: map_at_10 + value: 43.580999999999996 + - type: map_at_20 + value: 43.954 + - type: map_at_100 + value: 44.195 + - type: map_at_1000 + value: 44.255 - type: recall_at_1 - value: 0.24488 + value: 36.003 + - type: recall_at_3 + value: 48.533 + - type: recall_at_5 + value: 53.688 - type: recall_at_10 - value: 0.63659 + value: 60.111000000000004 + - type: recall_at_20 + value: 65.266 - type: recall_at_100 - value: 0.89702 + value: 74.941 - type: recall_at_1000 - value: 0.97996 + value: 89.056 + - type: precision_at_1 + value: 36.003 + - type: precision_at_3 + value: 16.178 + - type: precision_at_5 + value: 10.738 + - type: precision_at_10 + value: 6.010999999999999 + - type: precision_at_20 + value: 3.263 + - type: precision_at_100 + value: 0.749 + - type: precision_at_1000 + value: 0.089 + - type: mrr_at_1 + value: 36.0032 + - type: mrr_at_3 + value: 41.5015 + - type: mrr_at_5 + value: 42.695 + - type: mrr_at_10 + value: 43.580600000000004 + - type: mrr_at_20 + value: 43.9543 + - type: mrr_at_100 + value: 44.195299999999996 + - type: mrr_at_1000 + value: 44.255 + - type: nauc_ndcg_at_1_max + value: 32.9994 + - type: nauc_ndcg_at_1_std + value: -12.2575 + - type: nauc_ndcg_at_1_diff1 + value: 55.63360000000001 + - type: nauc_ndcg_at_3_max + value: 33.314899999999994 + - type: nauc_ndcg_at_3_std + value: -11.4208 + - type: nauc_ndcg_at_3_diff1 + value: 50.995599999999996 + - type: nauc_ndcg_at_5_max + value: 33.1612 + - type: nauc_ndcg_at_5_std + value: -11.4067 + - type: nauc_ndcg_at_5_diff1 + value: 50.766999999999996 + - type: nauc_ndcg_at_10_max + value: 32.903999999999996 + - type: nauc_ndcg_at_10_std + value: -11.447000000000001 + - type: nauc_ndcg_at_10_diff1 + value: 50.1061 + - type: nauc_ndcg_at_20_max + value: 32.8849 + - type: nauc_ndcg_at_20_std + value: -11.4567 + - type: nauc_ndcg_at_20_diff1 + value: 50.0131 + - type: nauc_ndcg_at_100_max + value: 32.5449 + - type: nauc_ndcg_at_100_std + value: -11.0686 + - type: nauc_ndcg_at_100_diff1 + value: 49.7046 + - type: nauc_ndcg_at_1000_max + value: 32.7575 + - type: nauc_ndcg_at_1000_std + value: -10.9682 + - type: nauc_ndcg_at_1000_diff1 + value: 50.17359999999999 + - type: nauc_map_at_1_max + value: 32.9994 + - type: nauc_map_at_1_std + value: -12.2575 + - type: nauc_map_at_1_diff1 + value: 55.63360000000001 + - type: nauc_map_at_3_max + value: 33.2746 + - type: nauc_map_at_3_std + value: -11.5215 + - type: nauc_map_at_3_diff1 + value: 52.1439 + - type: nauc_map_at_5_max + value: 33.206799999999994 + - type: nauc_map_at_5_std + value: -11.533 + - type: nauc_map_at_5_diff1 + value: 52.0477 + - type: nauc_map_at_10_max + value: 33.1113 + - type: nauc_map_at_10_std + value: -11.5406 + - type: nauc_map_at_10_diff1 + value: 51.8103 + - type: nauc_map_at_20_max + value: 33.070899999999995 + - type: nauc_map_at_20_std + value: -11.5655 + - type: nauc_map_at_20_diff1 + value: 51.7759 + - type: nauc_map_at_100_max + value: 32.9989 + - type: nauc_map_at_100_std + value: -11.546 + - type: nauc_map_at_100_diff1 + value: 51.739000000000004 + - type: nauc_map_at_1000_max + value: 33.0074 + - type: nauc_map_at_1000_std + value: -11.541 + - type: nauc_map_at_1000_diff1 + value: 51.7548 + - type: nauc_recall_at_1_max + value: 32.9994 + - type: nauc_recall_at_1_std + value: -12.2575 + - type: nauc_recall_at_1_diff1 + value: 55.63360000000001 + - type: nauc_recall_at_3_max + value: 33.4172 + - type: nauc_recall_at_3_std + value: -11.1701 + - type: nauc_recall_at_3_diff1 + value: 47.6442 + - type: nauc_recall_at_5_max + value: 32.962799999999994 + - type: nauc_recall_at_5_std + value: -11.0448 + - type: nauc_recall_at_5_diff1 + value: 46.8433 + - type: nauc_recall_at_10_max + value: 32.042500000000004 + - type: nauc_recall_at_10_std + value: -11.2125 + - type: nauc_recall_at_10_diff1 + value: 44.2396 + - type: nauc_recall_at_20_max + value: 32.1997 + - type: nauc_recall_at_20_std + value: -11.0222 + - type: nauc_recall_at_20_diff1 + value: 43.4014 + - type: nauc_recall_at_100_max + value: 29.972500000000004 + - type: nauc_recall_at_100_std + value: -7.2572 + - type: nauc_recall_at_100_diff1 + value: 39.285199999999996 + - type: nauc_recall_at_1000_max + value: 31.759300000000003 + - type: nauc_recall_at_1000_std + value: -1.555 + - type: nauc_recall_at_1000_diff1 + value: 38.7819 + - type: nauc_precision_at_1_max + value: 32.9994 + - type: nauc_precision_at_1_std + value: -12.2575 + - type: nauc_precision_at_1_diff1 + value: 55.63360000000001 + - type: nauc_precision_at_3_max + value: 33.4172 + - type: nauc_precision_at_3_std + value: -11.1701 + - type: nauc_precision_at_3_diff1 + value: 47.6442 + - type: nauc_precision_at_5_max + value: 32.962799999999994 + - type: nauc_precision_at_5_std + value: -11.0448 + - type: nauc_precision_at_5_diff1 + value: 46.8433 + - type: nauc_precision_at_10_max + value: 32.042500000000004 + - type: nauc_precision_at_10_std + value: -11.2125 + - type: nauc_precision_at_10_diff1 + value: 44.2396 + - type: nauc_precision_at_20_max + value: 32.1997 + - type: nauc_precision_at_20_std + value: -11.0222 + - type: nauc_precision_at_20_diff1 + value: 43.4014 + - type: nauc_precision_at_100_max + value: 29.972500000000004 + - type: nauc_precision_at_100_std + value: -7.2572 + - type: nauc_precision_at_100_diff1 + value: 39.285199999999996 + - type: nauc_precision_at_1000_max + value: 31.759300000000003 + - type: nauc_precision_at_1000_std + value: -1.555 + - type: nauc_precision_at_1000_diff1 + value: 38.7819 + - type: nauc_mrr_at_1_max + value: 33.1174 + - type: nauc_mrr_at_1_std + value: -12.0388 + - type: nauc_mrr_at_1_diff1 + value: 55.63360000000001 + - type: nauc_mrr_at_3_max + value: 33.333800000000004 + - type: nauc_mrr_at_3_std + value: -11.4119 + - type: nauc_mrr_at_3_diff1 + value: 52.1439 + - type: nauc_mrr_at_5_max + value: 33.2665 + - type: nauc_mrr_at_5_std + value: -11.4223 + - type: nauc_mrr_at_5_diff1 + value: 52.0477 + - type: nauc_mrr_at_10_max + value: 33.1716 + - type: nauc_mrr_at_10_std + value: -11.4289 + - type: nauc_mrr_at_10_diff1 + value: 51.8103 + - type: nauc_mrr_at_20_max + value: 33.1315 + - type: nauc_mrr_at_20_std + value: -11.4531 + - type: nauc_mrr_at_20_diff1 + value: 51.7759 + - type: nauc_mrr_at_100_max + value: 33.0598 + - type: nauc_mrr_at_100_std + value: -11.4331 + - type: nauc_mrr_at_100_diff1 + value: 51.739000000000004 + - type: nauc_mrr_at_1000_max + value: 33.0684 + - type: nauc_mrr_at_1000_std + value: -11.428 + - type: nauc_mrr_at_1000_diff1 + value: 51.7548 + - type: main_score + value: 47.549 + task: + type: Retrieval + - dataset: + config: java + name: MTEB CodeSearchNetCCRetrieval (java) + revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 + split: test + type: CoIR-Retrieval/CodeSearchNet-ccr + metrics: + - type: ndcg_at_1 + value: 33.355000000000004 + - type: ndcg_at_3 + value: 41.551 + - type: ndcg_at_5 + value: 43.592 + - type: ndcg_at_10 + value: 45.539 + - type: ndcg_at_20 + value: 46.922999999999995 + - type: ndcg_at_100 + value: 49.01 + - type: ndcg_at_1000 + value: 50.592000000000006 + - type: map_at_1 + value: 33.355000000000004 + - type: map_at_3 + value: 39.582 + - type: map_at_5 + value: 40.716 + - type: map_at_10 + value: 41.524 + - type: map_at_20 + value: 41.905 + - type: map_at_100 + value: 42.185 + - type: map_at_1000 + value: 42.239 + - type: recall_at_1 + value: 33.355000000000004 + - type: recall_at_3 + value: 47.23 + - type: recall_at_5 + value: 52.17699999999999 + - type: recall_at_10 + value: 58.17400000000001 - type: recall_at_20 - value: 0.72652 + value: 63.641999999999996 + - type: recall_at_100 + value: 75.034 + - type: recall_at_1000 + value: 87.85 + - type: precision_at_1 + value: 33.355000000000004 + - type: precision_at_3 + value: 15.742999999999999 + - type: precision_at_5 + value: 10.435 + - type: precision_at_10 + value: 5.817 + - type: precision_at_20 + value: 3.182 + - type: precision_at_100 + value: 0.75 + - type: precision_at_1000 + value: 0.08800000000000001 + - type: mrr_at_1 + value: 33.3455 + - type: mrr_at_3 + value: 39.569500000000005 + - type: mrr_at_5 + value: 40.7055 + - type: mrr_at_10 + value: 41.5123 + - type: mrr_at_20 + value: 41.8948 + - type: mrr_at_100 + value: 42.175200000000004 + - type: mrr_at_1000 + value: 42.228500000000004 + - type: nauc_ndcg_at_1_max + value: 29.177500000000002 + - type: nauc_ndcg_at_1_std + value: -5.8229999999999995 + - type: nauc_ndcg_at_1_diff1 + value: 53.2548 + - type: nauc_ndcg_at_3_max + value: 31.0728 + - type: nauc_ndcg_at_3_std + value: -4.3403 + - type: nauc_ndcg_at_3_diff1 + value: 48.6597 + - type: nauc_ndcg_at_5_max + value: 30.9135 + - type: nauc_ndcg_at_5_std + value: -3.5812999999999997 + - type: nauc_ndcg_at_5_diff1 + value: 47.6076 + - type: nauc_ndcg_at_10_max + value: 30.662899999999997 + - type: nauc_ndcg_at_10_std + value: -3.3078999999999996 + - type: nauc_ndcg_at_10_diff1 + value: 46.9647 + - type: nauc_ndcg_at_20_max + value: 30.7534 + - type: nauc_ndcg_at_20_std + value: -2.6957 + - type: nauc_ndcg_at_20_diff1 + value: 46.6956 + - type: nauc_ndcg_at_100_max + value: 30.8268 + - type: nauc_ndcg_at_100_std + value: -1.9675000000000002 + - type: nauc_ndcg_at_100_diff1 + value: 46.4854 + - type: nauc_ndcg_at_1000_max + value: 30.7713 + - type: nauc_ndcg_at_1000_std + value: -1.9892 + - type: nauc_ndcg_at_1000_diff1 + value: 46.7157 + - type: nauc_map_at_1_max + value: 29.177500000000002 + - type: nauc_map_at_1_std + value: -5.8229999999999995 + - type: nauc_map_at_1_diff1 + value: 53.2548 + - type: nauc_map_at_3_max + value: 30.6136 + - type: nauc_map_at_3_std + value: -4.7136 + - type: nauc_map_at_3_diff1 + value: 49.709399999999995 + - type: nauc_map_at_5_max + value: 30.523699999999998 + - type: nauc_map_at_5_std + value: -4.288200000000001 + - type: nauc_map_at_5_diff1 + value: 49.127700000000004 + - type: nauc_map_at_10_max + value: 30.4224 + - type: nauc_map_at_10_std + value: -4.1822 + - type: nauc_map_at_10_diff1 + value: 48.8812 + - type: nauc_map_at_20_max + value: 30.4446 + - type: nauc_map_at_20_std + value: -4.0194 + - type: nauc_map_at_20_diff1 + value: 48.8177 + - type: nauc_map_at_100_max + value: 30.4531 + - type: nauc_map_at_100_std + value: -3.9356 + - type: nauc_map_at_100_diff1 + value: 48.7971 + - type: nauc_map_at_1000_max + value: 30.4507 + - type: nauc_map_at_1000_std + value: -3.9337999999999997 + - type: nauc_map_at_1000_diff1 + value: 48.8055 + - type: nauc_recall_at_1_max + value: 29.177500000000002 + - type: nauc_recall_at_1_std + value: -5.8229999999999995 + - type: nauc_recall_at_1_diff1 + value: 53.2548 + - type: nauc_recall_at_3_max + value: 32.3983 + - type: nauc_recall_at_3_std + value: -3.2567 + - type: nauc_recall_at_3_diff1 + value: 45.6552 + - type: nauc_recall_at_5_max + value: 32.043 + - type: nauc_recall_at_5_std + value: -1.3823 + - type: nauc_recall_at_5_diff1 + value: 42.9898 + - type: nauc_recall_at_10_max + value: 31.272 + - type: nauc_recall_at_10_std + value: -0.3417 + - type: nauc_recall_at_10_diff1 + value: 40.5539 + - type: nauc_recall_at_20_max + value: 31.7395 + - type: nauc_recall_at_20_std + value: 2.645 + - type: nauc_recall_at_20_diff1 + value: 38.777499999999996 + - type: nauc_recall_at_100_max + value: 32.6198 + - type: nauc_recall_at_100_std + value: 10.1172 + - type: nauc_recall_at_100_diff1 + value: 34.6806 + - type: nauc_recall_at_1000_max + value: 33.0633 + - type: nauc_recall_at_1000_std + value: 19.5697 + - type: nauc_recall_at_1000_diff1 + value: 29.418699999999998 + - type: nauc_precision_at_1_max + value: 29.177500000000002 + - type: nauc_precision_at_1_std + value: -5.8229999999999995 + - type: nauc_precision_at_1_diff1 + value: 53.2548 + - type: nauc_precision_at_3_max + value: 32.3983 + - type: nauc_precision_at_3_std + value: -3.2567 + - type: nauc_precision_at_3_diff1 + value: 45.6552 + - type: nauc_precision_at_5_max + value: 32.043 + - type: nauc_precision_at_5_std + value: -1.3823 + - type: nauc_precision_at_5_diff1 + value: 42.9898 + - type: nauc_precision_at_10_max + value: 31.272 + - type: nauc_precision_at_10_std + value: -0.3417 + - type: nauc_precision_at_10_diff1 + value: 40.5539 + - type: nauc_precision_at_20_max + value: 31.7395 + - type: nauc_precision_at_20_std + value: 2.645 + - type: nauc_precision_at_20_diff1 + value: 38.777499999999996 + - type: nauc_precision_at_100_max + value: 32.6198 + - type: nauc_precision_at_100_std + value: 10.1172 + - type: nauc_precision_at_100_diff1 + value: 34.6806 + - type: nauc_precision_at_1000_max + value: 33.0633 + - type: nauc_precision_at_1000_std + value: 19.5697 + - type: nauc_precision_at_1000_diff1 + value: 29.418699999999998 + - type: nauc_mrr_at_1_max + value: 29.217900000000004 + - type: nauc_mrr_at_1_std + value: -5.8532 + - type: nauc_mrr_at_1_diff1 + value: 53.283100000000005 + - type: nauc_mrr_at_3_max + value: 30.6327 + - type: nauc_mrr_at_3_std + value: -4.7439 + - type: nauc_mrr_at_3_diff1 + value: 49.7477 + - type: nauc_mrr_at_5_max + value: 30.5427 + - type: nauc_mrr_at_5_std + value: -4.3167 + - type: nauc_mrr_at_5_diff1 + value: 49.152 + - type: nauc_mrr_at_10_max + value: 30.444100000000002 + - type: nauc_mrr_at_10_std + value: -4.2066 + - type: nauc_mrr_at_10_diff1 + value: 48.9038 + - type: nauc_mrr_at_20_max + value: 30.462899999999998 + - type: nauc_mrr_at_20_std + value: -4.0467 + - type: nauc_mrr_at_20_diff1 + value: 48.8397 + - type: nauc_mrr_at_100_max + value: 30.4714 + - type: nauc_mrr_at_100_std + value: -3.963 + - type: nauc_mrr_at_100_diff1 + value: 48.8192 + - type: nauc_mrr_at_1000_max + value: 30.469 + - type: nauc_mrr_at_1000_std + value: -3.9613 + - type: nauc_mrr_at_1000_diff1 + value: 48.8277 + - type: main_score + value: 45.539 + task: + type: Retrieval + - dataset: + config: php + name: MTEB CodeSearchNetCCRetrieval (php) + revision: 6e1effa2c03723c5fde48ee912b5ee08d4f211e8 + split: test + type: CoIR-Retrieval/CodeSearchNet-ccr + metrics: + - type: ndcg_at_1 + value: 25.139 + - type: ndcg_at_3 + value: 31.922 + - type: ndcg_at_5 + value: 33.989999999999995 + - type: ndcg_at_10 + value: 35.942 + - type: ndcg_at_20 + value: 37.506 + - type: ndcg_at_100 + value: 39.971000000000004 + - type: ndcg_at_1000 + value: 42.074 + - type: map_at_1 + value: 25.139 + - type: map_at_3 + value: 30.263 + - type: map_at_5 + value: 31.411 + - type: map_at_10 + value: 32.218 + - type: map_at_20 + value: 32.65 + - type: map_at_100 + value: 32.979 + - type: map_at_1000 + value: 33.050000000000004 + - type: recall_at_1 + value: 25.139 - type: recall_at_3 - value: 0.42827 + value: 36.720000000000006 - type: recall_at_5 - value: 0.52081 + value: 41.737 + - type: recall_at_10 + value: 47.766999999999996 + - type: recall_at_20 + value: 53.932 + - type: recall_at_100 + value: 67.38300000000001 + - type: recall_at_1000 + value: 84.416 + - type: precision_at_1 + value: 25.139 + - type: precision_at_3 + value: 12.24 + - type: precision_at_5 + value: 8.347 + - type: precision_at_10 + value: 4.777 + - type: precision_at_20 + value: 2.697 + - type: precision_at_100 + value: 0.674 + - type: precision_at_1000 + value: 0.084 + - type: mrr_at_1 + value: 25.1463 + - type: mrr_at_3 + value: 30.2709 + - type: mrr_at_5 + value: 31.4126 + - type: mrr_at_10 + value: 32.2202 + - type: mrr_at_20 + value: 32.6527 + - type: mrr_at_100 + value: 32.9822 + - type: mrr_at_1000 + value: 33.0527 + - type: nauc_ndcg_at_1_max + value: 24.082600000000003 + - type: nauc_ndcg_at_1_std + value: -3.9068 + - type: nauc_ndcg_at_1_diff1 + value: 50.1815 + - type: nauc_ndcg_at_3_max + value: 23.160700000000002 + - type: nauc_ndcg_at_3_std + value: -3.3746 + - type: nauc_ndcg_at_3_diff1 + value: 45.009 + - type: nauc_ndcg_at_5_max + value: 22.644000000000002 + - type: nauc_ndcg_at_5_std + value: -3.0027999999999997 + - type: nauc_ndcg_at_5_diff1 + value: 44.0016 + - type: nauc_ndcg_at_10_max + value: 22.3578 + - type: nauc_ndcg_at_10_std + value: -2.5096 + - type: nauc_ndcg_at_10_diff1 + value: 43.4367 + - type: nauc_ndcg_at_20_max + value: 22.0477 + - type: nauc_ndcg_at_20_std + value: -1.7484 + - type: nauc_ndcg_at_20_diff1 + value: 42.9771 + - type: nauc_ndcg_at_100_max + value: 21.7016 + - type: nauc_ndcg_at_100_std + value: -1.0854000000000001 + - type: nauc_ndcg_at_100_diff1 + value: 42.707 + - type: nauc_ndcg_at_1000_max + value: 21.988 + - type: nauc_ndcg_at_1000_std + value: -0.8564999999999999 + - type: nauc_ndcg_at_1000_diff1 + value: 43.0368 + - type: nauc_map_at_1_max + value: 24.082600000000003 + - type: nauc_map_at_1_std + value: -3.9068 + - type: nauc_map_at_1_diff1 + value: 50.1815 + - type: nauc_map_at_3_max + value: 23.418 + - type: nauc_map_at_3_std + value: -3.4922 + - type: nauc_map_at_3_diff1 + value: 46.19 + - type: nauc_map_at_5_max + value: 23.1203 + - type: nauc_map_at_5_std + value: -3.2856000000000005 + - type: nauc_map_at_5_diff1 + value: 45.6063 + - type: nauc_map_at_10_max + value: 23.0132 + - type: nauc_map_at_10_std + value: -3.0803000000000003 + - type: nauc_map_at_10_diff1 + value: 45.3708 + - type: nauc_map_at_20_max + value: 22.926199999999998 + - type: nauc_map_at_20_std + value: -2.8717 + - type: nauc_map_at_20_diff1 + value: 45.2482 + - type: nauc_map_at_100_max + value: 22.8776 + - type: nauc_map_at_100_std + value: -2.7819 + - type: nauc_map_at_100_diff1 + value: 45.2205 + - type: nauc_map_at_1000_max + value: 22.886 + - type: nauc_map_at_1000_std + value: -2.7714 + - type: nauc_map_at_1000_diff1 + value: 45.231300000000005 + - type: nauc_recall_at_1_max + value: 24.082600000000003 + - type: nauc_recall_at_1_std + value: -3.9068 + - type: nauc_recall_at_1_diff1 + value: 50.1815 + - type: nauc_recall_at_3_max + value: 22.442500000000003 + - type: nauc_recall_at_3_std + value: -3.0562 + - type: nauc_recall_at_3_diff1 + value: 41.797000000000004 + - type: nauc_recall_at_5_max + value: 21.2749 + - type: nauc_recall_at_5_std + value: -2.1853000000000002 + - type: nauc_recall_at_5_diff1 + value: 39.543 + - type: nauc_recall_at_10_max + value: 20.336399999999998 + - type: nauc_recall_at_10_std + value: -0.6941 + - type: nauc_recall_at_10_diff1 + value: 37.7835 + - type: nauc_recall_at_20_max + value: 19.031799999999997 + - type: nauc_recall_at_20_std + value: 2.4044 + - type: nauc_recall_at_20_diff1 + value: 35.6973 + - type: nauc_recall_at_100_max + value: 16.1657 + - type: nauc_recall_at_100_std + value: 7.480199999999999 + - type: nauc_recall_at_100_diff1 + value: 32.2845 + - type: nauc_recall_at_1000_max + value: 16.6175 + - type: nauc_recall_at_1000_std + value: 17.7626 + - type: nauc_recall_at_1000_diff1 + value: 29.4846 + - type: nauc_precision_at_1_max + value: 24.082600000000003 + - type: nauc_precision_at_1_std + value: -3.9068 + - type: nauc_precision_at_1_diff1 + value: 50.1815 + - type: nauc_precision_at_3_max + value: 22.442500000000003 + - type: nauc_precision_at_3_std + value: -3.0562 + - type: nauc_precision_at_3_diff1 + value: 41.797000000000004 + - type: nauc_precision_at_5_max + value: 21.2749 + - type: nauc_precision_at_5_std + value: -2.1853000000000002 + - type: nauc_precision_at_5_diff1 + value: 39.543 + - type: nauc_precision_at_10_max + value: 20.336399999999998 + - type: nauc_precision_at_10_std + value: -0.6941 + - type: nauc_precision_at_10_diff1 + value: 37.7835 + - type: nauc_precision_at_20_max + value: 19.031799999999997 + - type: nauc_precision_at_20_std + value: 2.4044 + - type: nauc_precision_at_20_diff1 + value: 35.6973 + - type: nauc_precision_at_100_max + value: 16.1657 + - type: nauc_precision_at_100_std + value: 7.480199999999999 + - type: nauc_precision_at_100_diff1 + value: 32.2845 + - type: nauc_precision_at_1000_max + value: 16.6175 + - type: nauc_precision_at_1000_std + value: 17.7626 + - type: nauc_precision_at_1000_diff1 + value: 29.4846 + - type: nauc_mrr_at_1_max + value: 23.9848 + - type: nauc_mrr_at_1_std + value: -3.9669000000000003 + - type: nauc_mrr_at_1_diff1 + value: 50.152699999999996 + - type: nauc_mrr_at_3_max + value: 23.3397 + - type: nauc_mrr_at_3_std + value: -3.5128 + - type: nauc_mrr_at_3_diff1 + value: 46.1227 + - type: nauc_mrr_at_5_max + value: 23.0454 + - type: nauc_mrr_at_5_std + value: -3.3141 + - type: nauc_mrr_at_5_diff1 + value: 45.561 + - type: nauc_mrr_at_10_max + value: 22.9526 + - type: nauc_mrr_at_10_std + value: -3.1052 + - type: nauc_mrr_at_10_diff1 + value: 45.3316 + - type: nauc_mrr_at_20_max + value: 22.8654 + - type: nauc_mrr_at_20_std + value: -2.8967 + - type: nauc_mrr_at_20_diff1 + value: 45.2089 + - type: nauc_mrr_at_100_max + value: 22.8164 + - type: nauc_mrr_at_100_std + value: -2.8074000000000003 + - type: nauc_mrr_at_100_diff1 + value: 45.1812 + - type: nauc_mrr_at_1000_max + value: 22.8248 + - type: nauc_mrr_at_1000_std + value: -2.7968 + - type: nauc_mrr_at_1000_diff1 + value: 45.191900000000004 + - type: main_score + value: 35.942 + task: + type: Retrieval + - dataset: + config: python + name: MTEB CodeSearchNetRetrieval (python) + revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 + split: test + type: code-search-net/code_search_net + metrics: + - type: ndcg_at_1 + value: 70.89999999999999 + - type: ndcg_at_3 + value: 80.06400000000001 + - type: ndcg_at_5 + value: 81.703 + - type: ndcg_at_10 + value: 83.12 + - type: ndcg_at_20 + value: 83.67999999999999 + - type: ndcg_at_100 + value: 84.11 + - type: ndcg_at_1000 + value: 84.195 + - type: map_at_1 + value: 70.89999999999999 + - type: map_at_3 + value: 77.86699999999999 + - type: map_at_5 + value: 78.77199999999999 + - type: map_at_10 + value: 79.353 + - type: map_at_20 + value: 79.508 + - type: map_at_100 + value: 79.569 + - type: map_at_1000 + value: 79.571 + - type: recall_at_1 + value: 70.89999999999999 + - type: recall_at_3 + value: 86.4 + - type: recall_at_5 + value: 90.4 + - type: recall_at_10 + value: 94.8 + - type: recall_at_20 + value: 97.0 + - type: recall_at_100 + value: 99.3 + - type: recall_at_1000 + value: 100.0 + - type: precision_at_1 + value: 70.89999999999999 + - type: precision_at_3 + value: 28.799999999999997 + - type: precision_at_5 + value: 18.08 + - type: precision_at_10 + value: 9.48 + - type: precision_at_20 + value: 4.8500000000000005 + - type: precision_at_100 + value: 0.993 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 70.89999999999999 + - type: mrr_at_3 + value: 77.8667 + - type: mrr_at_5 + value: 78.7717 + - type: mrr_at_10 + value: 79.3526 + - type: mrr_at_20 + value: 79.5084 + - type: mrr_at_100 + value: 79.5687 + - type: mrr_at_1000 + value: 79.5713 + - type: nauc_ndcg_at_1_max + value: 42.7162 + - type: nauc_ndcg_at_1_std + value: -4.6818 + - type: nauc_ndcg_at_1_diff1 + value: 70.6364 + - type: nauc_ndcg_at_3_max + value: 48.1282 + - type: nauc_ndcg_at_3_std + value: -2.8091 + - type: nauc_ndcg_at_3_diff1 + value: 67.9426 + - type: nauc_ndcg_at_5_max + value: 45.713 + - type: nauc_ndcg_at_5_std + value: -4.0022 + - type: nauc_ndcg_at_5_diff1 + value: 67.0684 + - type: nauc_ndcg_at_10_max + value: 45.8762 + - type: nauc_ndcg_at_10_std + value: -2.8594999999999997 + - type: nauc_ndcg_at_10_diff1 + value: 67.318 + - type: nauc_ndcg_at_20_max + value: 45.8448 + - type: nauc_ndcg_at_20_std + value: -2.9843 + - type: nauc_ndcg_at_20_diff1 + value: 67.5016 + - type: nauc_ndcg_at_100_max + value: 45.9045 + - type: nauc_ndcg_at_100_std + value: -3.1647000000000003 + - type: nauc_ndcg_at_100_diff1 + value: 67.8211 + - type: nauc_ndcg_at_1000_max + value: 45.7011 + - type: nauc_ndcg_at_1000_std + value: -3.4981 + - type: nauc_ndcg_at_1000_diff1 + value: 67.9137 + - type: nauc_map_at_1_max + value: 42.7162 + - type: nauc_map_at_1_std + value: -4.6818 + - type: nauc_map_at_1_diff1 + value: 70.6364 + - type: nauc_map_at_3_max + value: 46.5287 + - type: nauc_map_at_3_std + value: -3.6239 + - type: nauc_map_at_3_diff1 + value: 68.5879 + - type: nauc_map_at_5_max + value: 45.291599999999995 + - type: nauc_map_at_5_std + value: -4.2172 + - type: nauc_map_at_5_diff1 + value: 68.1788 + - type: nauc_map_at_10_max + value: 45.31 + - type: nauc_map_at_10_std + value: -3.8557 + - type: nauc_map_at_10_diff1 + value: 68.2538 + - type: nauc_map_at_20_max + value: 45.2841 + - type: nauc_map_at_20_std + value: -3.92 + - type: nauc_map_at_20_diff1 + value: 68.2978 + - type: nauc_map_at_100_max + value: 45.3154 + - type: nauc_map_at_100_std + value: -3.929 + - type: nauc_map_at_100_diff1 + value: 68.3362 + - type: nauc_map_at_1000_max + value: 45.3097 + - type: nauc_map_at_1000_std + value: -3.9364999999999997 + - type: nauc_map_at_1000_diff1 + value: 68.3376 + - type: nauc_recall_at_1_max + value: 42.7162 + - type: nauc_recall_at_1_std + value: -4.6818 + - type: nauc_recall_at_1_diff1 + value: 70.6364 + - type: nauc_recall_at_3_max + value: 55.0798 + - type: nauc_recall_at_3_std + value: 0.9014 + - type: nauc_recall_at_3_diff1 + value: 65.2358 + - type: nauc_recall_at_5_max + value: 47.4148 + - type: nauc_recall_at_5_std + value: -2.9387 + - type: nauc_recall_at_5_diff1 + value: 60.644299999999994 + - type: nauc_recall_at_10_max + value: 50.820600000000006 + - type: nauc_recall_at_10_std + value: 8.7499 + - type: nauc_recall_at_10_diff1 + value: 58.34049999999999 + - type: nauc_recall_at_20_max + value: 54.4382 + - type: nauc_recall_at_20_std + value: 16.0862 + - type: nauc_recall_at_20_diff1 + value: 55.5229 + - type: nauc_recall_at_100_max + value: 79.2317 + - type: nauc_recall_at_100_std + value: 54.095000000000006 + - type: nauc_recall_at_100_diff1 + value: 50.6869 + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_precision_at_1_max + value: 42.7162 + - type: nauc_precision_at_1_std + value: -4.6818 + - type: nauc_precision_at_1_diff1 + value: 70.6364 + - type: nauc_precision_at_3_max + value: 55.0798 + - type: nauc_precision_at_3_std + value: 0.9014 + - type: nauc_precision_at_3_diff1 + value: 65.2358 + - type: nauc_precision_at_5_max + value: 47.4148 + - type: nauc_precision_at_5_std + value: -2.9387 + - type: nauc_precision_at_5_diff1 + value: 60.644299999999994 + - type: nauc_precision_at_10_max + value: 50.820600000000006 + - type: nauc_precision_at_10_std + value: 8.7499 + - type: nauc_precision_at_10_diff1 + value: 58.34049999999999 + - type: nauc_precision_at_20_max + value: 54.4382 + - type: nauc_precision_at_20_std + value: 16.0862 + - type: nauc_precision_at_20_diff1 + value: 55.5229 + - type: nauc_precision_at_100_max + value: 79.2317 + - type: nauc_precision_at_100_std + value: 54.095000000000006 + - type: nauc_precision_at_100_diff1 + value: 50.6869 + - type: nauc_precision_at_1000_max + value: .nan + - type: nauc_precision_at_1000_std + value: .nan + - type: nauc_precision_at_1000_diff1 + value: .nan + - type: nauc_mrr_at_1_max + value: 42.7162 + - type: nauc_mrr_at_1_std + value: -4.6818 + - type: nauc_mrr_at_1_diff1 + value: 70.6364 + - type: nauc_mrr_at_3_max + value: 46.5287 + - type: nauc_mrr_at_3_std + value: -3.6239 + - type: nauc_mrr_at_3_diff1 + value: 68.5879 + - type: nauc_mrr_at_5_max + value: 45.291599999999995 + - type: nauc_mrr_at_5_std + value: -4.2172 + - type: nauc_mrr_at_5_diff1 + value: 68.1788 + - type: nauc_mrr_at_10_max + value: 45.31 + - type: nauc_mrr_at_10_std + value: -3.8557 + - type: nauc_mrr_at_10_diff1 + value: 68.2538 + - type: nauc_mrr_at_20_max + value: 45.2841 + - type: nauc_mrr_at_20_std + value: -3.92 + - type: nauc_mrr_at_20_diff1 + value: 68.2978 + - type: nauc_mrr_at_100_max + value: 45.3154 + - type: nauc_mrr_at_100_std + value: -3.929 + - type: nauc_mrr_at_100_diff1 + value: 68.3362 + - type: nauc_mrr_at_1000_max + value: 45.3097 + - type: nauc_mrr_at_1000_std + value: -3.9364999999999997 + - type: nauc_mrr_at_1000_diff1 + value: 68.3376 + - type: main_score + value: 83.12 + task: + type: Retrieval + - dataset: + config: javascript + name: MTEB CodeSearchNetRetrieval (javascript) + revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 + split: test + type: code-search-net/code_search_net + metrics: + - type: ndcg_at_1 + value: 57.99999999999999 + - type: ndcg_at_3 + value: 67.24900000000001 + - type: ndcg_at_5 + value: 68.781 + - type: ndcg_at_10 + value: 70.34 + - type: ndcg_at_20 + value: 71.24000000000001 + - type: ndcg_at_100 + value: 72.617 + - type: ndcg_at_1000 + value: 73.436 + - type: map_at_1 + value: 57.99999999999999 + - type: map_at_3 + value: 64.983 + - type: map_at_5 + value: 65.838 + - type: map_at_10 + value: 66.50500000000001 + - type: map_at_20 + value: 66.74600000000001 + - type: map_at_100 + value: 66.93299999999999 + - type: map_at_1000 + value: 66.959 + - type: recall_at_1 + value: 57.99999999999999 + - type: recall_at_3 + value: 73.8 + - type: recall_at_5 + value: 77.5 + - type: recall_at_10 + value: 82.19999999999999 + - type: recall_at_20 + value: 85.8 + - type: recall_at_100 + value: 93.30000000000001 + - type: recall_at_1000 + value: 100.0 + - type: precision_at_1 + value: 57.99999999999999 + - type: precision_at_3 + value: 24.6 + - type: precision_at_5 + value: 15.5 + - type: precision_at_10 + value: 8.219999999999999 + - type: precision_at_20 + value: 4.29 + - type: precision_at_100 + value: 0.9329999999999999 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 57.99999999999999 + - type: mrr_at_3 + value: 64.9833 + - type: mrr_at_5 + value: 65.8383 + - type: mrr_at_10 + value: 66.50500000000001 + - type: mrr_at_20 + value: 66.7464 + - type: mrr_at_100 + value: 66.9326 + - type: mrr_at_1000 + value: 66.9593 + - type: nauc_ndcg_at_1_max + value: 51.0918 + - type: nauc_ndcg_at_1_std + value: 12.0501 + - type: nauc_ndcg_at_1_diff1 + value: 69.1716 + - type: nauc_ndcg_at_3_max + value: 59.404199999999996 + - type: nauc_ndcg_at_3_std + value: 22.4787 + - type: nauc_ndcg_at_3_diff1 + value: 66.2602 + - type: nauc_ndcg_at_5_max + value: 60.711000000000006 + - type: nauc_ndcg_at_5_std + value: 24.1272 + - type: nauc_ndcg_at_5_diff1 + value: 65.9406 + - type: nauc_ndcg_at_10_max + value: 61.492599999999996 + - type: nauc_ndcg_at_10_std + value: 26.6758 + - type: nauc_ndcg_at_10_diff1 + value: 66.1164 + - type: nauc_ndcg_at_20_max + value: 61.34610000000001 + - type: nauc_ndcg_at_20_std + value: 27.331 + - type: nauc_ndcg_at_20_diff1 + value: 66.981 + - type: nauc_ndcg_at_100_max + value: 60.50020000000001 + - type: nauc_ndcg_at_100_std + value: 26.623 + - type: nauc_ndcg_at_100_diff1 + value: 66.4658 + - type: nauc_ndcg_at_1000_max + value: 59.600500000000004 + - type: nauc_ndcg_at_1000_std + value: 24.3596 + - type: nauc_ndcg_at_1000_diff1 + value: 66.7619 + - type: nauc_map_at_1_max + value: 51.0918 + - type: nauc_map_at_1_std + value: 12.0501 + - type: nauc_map_at_1_diff1 + value: 69.1716 + - type: nauc_map_at_3_max + value: 57.2093 + - type: nauc_map_at_3_std + value: 19.4523 + - type: nauc_map_at_3_diff1 + value: 67.0065 + - type: nauc_map_at_5_max + value: 57.81699999999999 + - type: nauc_map_at_5_std + value: 20.2597 + - type: nauc_map_at_5_diff1 + value: 66.8577 + - type: nauc_map_at_10_max + value: 58.052099999999996 + - type: nauc_map_at_10_std + value: 21.195 + - type: nauc_map_at_10_diff1 + value: 66.9095 + - type: nauc_map_at_20_max + value: 57.9955 + - type: nauc_map_at_20_std + value: 21.3121 + - type: nauc_map_at_20_diff1 + value: 67.1257 + - type: nauc_map_at_100_max + value: 57.8974 + - type: nauc_map_at_100_std + value: 21.2576 + - type: nauc_map_at_100_diff1 + value: 67.0765 + - type: nauc_map_at_1000_max + value: 57.873799999999996 + - type: nauc_map_at_1000_std + value: 21.195 + - type: nauc_map_at_1000_diff1 + value: 67.08579999999999 + - type: nauc_recall_at_1_max + value: 51.0918 + - type: nauc_recall_at_1_std + value: 12.0501 + - type: nauc_recall_at_1_diff1 + value: 69.1716 + - type: nauc_recall_at_3_max + value: 67.0934 + - type: nauc_recall_at_3_std + value: 33.2241 + - type: nauc_recall_at_3_diff1 + value: 63.65769999999999 + - type: nauc_recall_at_5_max + value: 72.2191 + - type: nauc_recall_at_5_std + value: 39.5657 + - type: nauc_recall_at_5_diff1 + value: 62.3367 + - type: nauc_recall_at_10_max + value: 78.3358 + - type: nauc_recall_at_10_std + value: 54.093599999999995 + - type: nauc_recall_at_10_diff1 + value: 62.605900000000005 + - type: nauc_recall_at_20_max + value: 81.0991 + - type: nauc_recall_at_20_std + value: 64.9068 + - type: nauc_recall_at_20_diff1 + value: 67.7761 + - type: nauc_recall_at_100_max + value: 85.0279 + - type: nauc_recall_at_100_std + value: 87.47930000000001 + - type: nauc_recall_at_100_diff1 + value: 58.818000000000005 + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_precision_at_1_max + value: 51.0918 + - type: nauc_precision_at_1_std + value: 12.0501 + - type: nauc_precision_at_1_diff1 + value: 69.1716 + - type: nauc_precision_at_3_max + value: 67.0934 + - type: nauc_precision_at_3_std + value: 33.2241 + - type: nauc_precision_at_3_diff1 + value: 63.65769999999999 + - type: nauc_precision_at_5_max + value: 72.2191 + - type: nauc_precision_at_5_std + value: 39.5657 + - type: nauc_precision_at_5_diff1 + value: 62.3367 + - type: nauc_precision_at_10_max + value: 78.3358 + - type: nauc_precision_at_10_std + value: 54.093599999999995 + - type: nauc_precision_at_10_diff1 + value: 62.605900000000005 + - type: nauc_precision_at_20_max + value: 81.0991 + - type: nauc_precision_at_20_std + value: 64.9068 + - type: nauc_precision_at_20_diff1 + value: 67.7761 + - type: nauc_precision_at_100_max + value: 85.0279 + - type: nauc_precision_at_100_std + value: 87.47930000000001 + - type: nauc_precision_at_100_diff1 + value: 58.818000000000005 + - type: nauc_precision_at_1000_max + value: .nan + - type: nauc_precision_at_1000_std + value: .nan + - type: nauc_precision_at_1000_diff1 + value: .nan + - type: nauc_mrr_at_1_max + value: 51.0918 + - type: nauc_mrr_at_1_std + value: 12.0501 + - type: nauc_mrr_at_1_diff1 + value: 69.1716 + - type: nauc_mrr_at_3_max + value: 57.2093 + - type: nauc_mrr_at_3_std + value: 19.4523 + - type: nauc_mrr_at_3_diff1 + value: 67.0065 + - type: nauc_mrr_at_5_max + value: 57.81699999999999 + - type: nauc_mrr_at_5_std + value: 20.2597 + - type: nauc_mrr_at_5_diff1 + value: 66.8577 + - type: nauc_mrr_at_10_max + value: 58.052099999999996 + - type: nauc_mrr_at_10_std + value: 21.195 + - type: nauc_mrr_at_10_diff1 + value: 66.9095 + - type: nauc_mrr_at_20_max + value: 57.9955 + - type: nauc_mrr_at_20_std + value: 21.3121 + - type: nauc_mrr_at_20_diff1 + value: 67.1257 + - type: nauc_mrr_at_100_max + value: 57.8974 + - type: nauc_mrr_at_100_std + value: 21.2576 + - type: nauc_mrr_at_100_diff1 + value: 67.0765 + - type: nauc_mrr_at_1000_max + value: 57.873799999999996 + - type: nauc_mrr_at_1000_std + value: 21.195 + - type: nauc_mrr_at_1000_diff1 + value: 67.08579999999999 + - type: main_score + value: 70.34 + task: + type: Retrieval + - dataset: + config: go + name: MTEB CodeSearchNetRetrieval (go) + revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 + split: test + type: code-search-net/code_search_net + metrics: + - type: ndcg_at_1 + value: 75.6 + - type: ndcg_at_3 + value: 84.112 + - type: ndcg_at_5 + value: 85.351 + - type: ndcg_at_10 + value: 86.139 + - type: ndcg_at_20 + value: 86.599 + - type: ndcg_at_100 + value: 86.971 + - type: ndcg_at_1000 + value: 87.086 + - type: map_at_1 + value: 75.6 + - type: map_at_3 + value: 82.1 + - type: map_at_5 + value: 82.78999999999999 + - type: map_at_10 + value: 83.122 + - type: map_at_20 + value: 83.25099999999999 + - type: map_at_100 + value: 83.30300000000001 + - type: map_at_1000 + value: 83.307 + - type: recall_at_1 + value: 75.6 + - type: recall_at_3 + value: 89.9 + - type: recall_at_5 + value: 92.9 + - type: recall_at_10 + value: 95.3 + - type: recall_at_20 + value: 97.1 + - type: recall_at_100 + value: 99.1 + - type: recall_at_1000 + value: 100.0 + - type: precision_at_1 + value: 75.6 + - type: precision_at_3 + value: 29.967 + - type: precision_at_5 + value: 18.58 + - type: precision_at_10 + value: 9.53 + - type: precision_at_20 + value: 4.855 + - type: precision_at_100 + value: 0.991 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 75.6 + - type: mrr_at_3 + value: 82.1 + - type: mrr_at_5 + value: 82.78999999999999 + - type: mrr_at_10 + value: 83.12230000000001 + - type: mrr_at_20 + value: 83.2511 + - type: mrr_at_100 + value: 83.3027 + - type: mrr_at_1000 + value: 83.307 + - type: nauc_ndcg_at_1_max + value: 50.9856 + - type: nauc_ndcg_at_1_std + value: 6.729 + - type: nauc_ndcg_at_1_diff1 + value: 75.68589999999999 + - type: nauc_ndcg_at_3_max + value: 59.266 + - type: nauc_ndcg_at_3_std + value: 10.0957 + - type: nauc_ndcg_at_3_diff1 + value: 73.3044 + - type: nauc_ndcg_at_5_max + value: 58.7545 + - type: nauc_ndcg_at_5_std + value: 9.295399999999999 + - type: nauc_ndcg_at_5_diff1 + value: 73.9355 + - type: nauc_ndcg_at_10_max + value: 58.7538 + - type: nauc_ndcg_at_10_std + value: 10.335999999999999 + - type: nauc_ndcg_at_10_diff1 + value: 74.01870000000001 + - type: nauc_ndcg_at_20_max + value: 57.9057 + - type: nauc_ndcg_at_20_std + value: 10.115300000000001 + - type: nauc_ndcg_at_20_diff1 + value: 74.456 + - type: nauc_ndcg_at_100_max + value: 57.198800000000006 + - type: nauc_ndcg_at_100_std + value: 9.2269 + - type: nauc_ndcg_at_100_diff1 + value: 74.2418 + - type: nauc_ndcg_at_1000_max + value: 57.1141 + - type: nauc_ndcg_at_1000_std + value: 9.366900000000001 + - type: nauc_ndcg_at_1000_diff1 + value: 74.3329 + - type: nauc_map_at_1_max + value: 50.9856 + - type: nauc_map_at_1_std + value: 6.729 + - type: nauc_map_at_1_diff1 + value: 75.68589999999999 + - type: nauc_map_at_3_max + value: 57.0017 + - type: nauc_map_at_3_std + value: 9.2059 + - type: nauc_map_at_3_diff1 + value: 73.9956 + - type: nauc_map_at_5_max + value: 56.6856 + - type: nauc_map_at_5_std + value: 8.8058 + - type: nauc_map_at_5_diff1 + value: 74.3367 + - type: nauc_map_at_10_max + value: 56.652100000000004 + - type: nauc_map_at_10_std + value: 9.1465 + - type: nauc_map_at_10_diff1 + value: 74.37519999999999 + - type: nauc_map_at_20_max + value: 56.4431 + - type: nauc_map_at_20_std + value: 9.0962 + - type: nauc_map_at_20_diff1 + value: 74.4763 + - type: nauc_map_at_100_max + value: 56.3572 + - type: nauc_map_at_100_std + value: 8.9981 + - type: nauc_map_at_100_diff1 + value: 74.4551 + - type: nauc_map_at_1000_max + value: 56.3527 + - type: nauc_map_at_1000_std + value: 9.0022 + - type: nauc_map_at_1000_diff1 + value: 74.4583 + - type: nauc_recall_at_1_max + value: 50.9856 + - type: nauc_recall_at_1_std + value: 6.729 + - type: nauc_recall_at_1_diff1 + value: 75.68589999999999 + - type: nauc_recall_at_3_max + value: 69.7291 + - type: nauc_recall_at_3_std + value: 14.183000000000002 + - type: nauc_recall_at_3_diff1 + value: 70.07900000000001 + - type: nauc_recall_at_5_max + value: 71.5009 + - type: nauc_recall_at_5_std + value: 11.9764 + - type: nauc_recall_at_5_diff1 + value: 71.5765 + - type: nauc_recall_at_10_max + value: 77.7927 + - type: nauc_recall_at_10_std + value: 22.2123 + - type: nauc_recall_at_10_diff1 + value: 71.0601 + - type: nauc_recall_at_20_max + value: 75.421 + - type: nauc_recall_at_20_std + value: 25.5385 + - type: nauc_recall_at_20_diff1 + value: 76.5318 + - type: nauc_recall_at_100_max + value: 64.4206 + - type: nauc_recall_at_100_std + value: -4.8864 + - type: nauc_recall_at_100_diff1 + value: 65.2765 + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_precision_at_1_max + value: 50.9856 + - type: nauc_precision_at_1_std + value: 6.729 + - type: nauc_precision_at_1_diff1 + value: 75.68589999999999 + - type: nauc_precision_at_3_max + value: 69.7291 + - type: nauc_precision_at_3_std + value: 14.183000000000002 + - type: nauc_precision_at_3_diff1 + value: 70.07900000000001 + - type: nauc_precision_at_5_max + value: 71.5009 + - type: nauc_precision_at_5_std + value: 11.9764 + - type: nauc_precision_at_5_diff1 + value: 71.5765 + - type: nauc_precision_at_10_max + value: 77.7927 + - type: nauc_precision_at_10_std + value: 22.2123 + - type: nauc_precision_at_10_diff1 + value: 71.0601 + - type: nauc_precision_at_20_max + value: 75.421 + - type: nauc_precision_at_20_std + value: 25.5385 + - type: nauc_precision_at_20_diff1 + value: 76.5318 + - type: nauc_precision_at_100_max + value: 64.4206 + - type: nauc_precision_at_100_std + value: -4.8864 + - type: nauc_precision_at_100_diff1 + value: 65.2765 + - type: nauc_precision_at_1000_max + value: .nan + - type: nauc_precision_at_1000_std + value: .nan + - type: nauc_precision_at_1000_diff1 + value: .nan + - type: nauc_mrr_at_1_max + value: 50.9856 + - type: nauc_mrr_at_1_std + value: 6.729 + - type: nauc_mrr_at_1_diff1 + value: 75.68589999999999 + - type: nauc_mrr_at_3_max + value: 57.0017 + - type: nauc_mrr_at_3_std + value: 9.2059 + - type: nauc_mrr_at_3_diff1 + value: 73.9956 + - type: nauc_mrr_at_5_max + value: 56.6856 + - type: nauc_mrr_at_5_std + value: 8.8058 + - type: nauc_mrr_at_5_diff1 + value: 74.3367 + - type: nauc_mrr_at_10_max + value: 56.652100000000004 + - type: nauc_mrr_at_10_std + value: 9.1465 + - type: nauc_mrr_at_10_diff1 + value: 74.37519999999999 + - type: nauc_mrr_at_20_max + value: 56.4431 + - type: nauc_mrr_at_20_std + value: 9.0962 + - type: nauc_mrr_at_20_diff1 + value: 74.4763 + - type: nauc_mrr_at_100_max + value: 56.3572 + - type: nauc_mrr_at_100_std + value: 8.9981 + - type: nauc_mrr_at_100_diff1 + value: 74.4551 + - type: nauc_mrr_at_1000_max + value: 56.3527 + - type: nauc_mrr_at_1000_std + value: 9.0022 + - type: nauc_mrr_at_1000_diff1 + value: 74.4583 + - type: main_score + value: 86.139 + task: + type: Retrieval + - dataset: + config: ruby + name: MTEB CodeSearchNetRetrieval (ruby) + revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 + split: test + type: code-search-net/code_search_net + metrics: + - type: ndcg_at_1 + value: 61.3 + - type: ndcg_at_3 + value: 71.232 + - type: ndcg_at_5 + value: 73.1 + - type: ndcg_at_10 + value: 74.736 + - type: ndcg_at_20 + value: 75.511 + - type: ndcg_at_100 + value: 76.416 + - type: ndcg_at_1000 + value: 76.996 + - type: map_at_1 + value: 61.3 + - type: map_at_3 + value: 68.85 + - type: map_at_5 + value: 69.895 + - type: map_at_10 + value: 70.581 + - type: map_at_20 + value: 70.80199999999999 + - type: map_at_100 + value: 70.94200000000001 + - type: map_at_1000 + value: 70.961 + - type: recall_at_1 + value: 61.3 + - type: recall_at_3 + value: 78.10000000000001 + - type: recall_at_5 + value: 82.6 + - type: recall_at_10 + value: 87.6 + - type: recall_at_20 + value: 90.60000000000001 + - type: recall_at_100 + value: 95.3 + - type: recall_at_1000 + value: 100.0 + - type: precision_at_1 + value: 61.3 + - type: precision_at_3 + value: 26.033 + - type: precision_at_5 + value: 16.520000000000003 + - type: precision_at_10 + value: 8.76 + - type: precision_at_20 + value: 4.53 + - type: precision_at_100 + value: 0.9530000000000001 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 61.3 + - type: mrr_at_3 + value: 68.85 + - type: mrr_at_5 + value: 69.895 + - type: mrr_at_10 + value: 70.58109999999999 + - type: mrr_at_20 + value: 70.8024 + - type: mrr_at_100 + value: 70.94160000000001 + - type: mrr_at_1000 + value: 70.96090000000001 + - type: nauc_ndcg_at_1_max + value: 54.2597 + - type: nauc_ndcg_at_1_std + value: 9.9915 + - type: nauc_ndcg_at_1_diff1 + value: 72.0029 + - type: nauc_ndcg_at_3_max + value: 58.517799999999994 + - type: nauc_ndcg_at_3_std + value: 13.256599999999999 + - type: nauc_ndcg_at_3_diff1 + value: 67.861 + - type: nauc_ndcg_at_5_max + value: 59.1541 + - type: nauc_ndcg_at_5_std + value: 16.237099999999998 + - type: nauc_ndcg_at_5_diff1 + value: 67.8155 + - type: nauc_ndcg_at_10_max + value: 59.1703 + - type: nauc_ndcg_at_10_std + value: 17.8202 + - type: nauc_ndcg_at_10_diff1 + value: 67.6082 + - type: nauc_ndcg_at_20_max + value: 58.829299999999996 + - type: nauc_ndcg_at_20_std + value: 18.001900000000003 + - type: nauc_ndcg_at_20_diff1 + value: 67.6747 + - type: nauc_ndcg_at_100_max + value: 58.675399999999996 + - type: nauc_ndcg_at_100_std + value: 17.7394 + - type: nauc_ndcg_at_100_diff1 + value: 68.02810000000001 + - type: nauc_ndcg_at_1000_max + value: 58.333400000000005 + - type: nauc_ndcg_at_1000_std + value: 16.169900000000002 + - type: nauc_ndcg_at_1000_diff1 + value: 68.3788 + - type: nauc_map_at_1_max + value: 54.2597 + - type: nauc_map_at_1_std + value: 9.9915 + - type: nauc_map_at_1_diff1 + value: 72.0029 + - type: nauc_map_at_3_max + value: 57.4277 + - type: nauc_map_at_3_std + value: 12.1778 + - type: nauc_map_at_3_diff1 + value: 69.0312 + - type: nauc_map_at_5_max + value: 57.7291 + - type: nauc_map_at_5_std + value: 13.655800000000001 + - type: nauc_map_at_5_diff1 + value: 69.0376 + - type: nauc_map_at_10_max + value: 57.7091 + - type: nauc_map_at_10_std + value: 14.2236 + - type: nauc_map_at_10_diff1 + value: 68.99849999999999 + - type: nauc_map_at_20_max + value: 57.605700000000006 + - type: nauc_map_at_20_std + value: 14.2305 + - type: nauc_map_at_20_diff1 + value: 69.0304 + - type: nauc_map_at_100_max + value: 57.6007 + - type: nauc_map_at_100_std + value: 14.219499999999998 + - type: nauc_map_at_100_diff1 + value: 69.0682 + - type: nauc_map_at_1000_max + value: 57.5939 + - type: nauc_map_at_1000_std + value: 14.1793 + - type: nauc_map_at_1000_diff1 + value: 69.0767 + - type: nauc_recall_at_1_max + value: 54.2597 + - type: nauc_recall_at_1_std + value: 9.9915 + - type: nauc_recall_at_1_diff1 + value: 72.0029 + - type: nauc_recall_at_3_max + value: 62.5301 + - type: nauc_recall_at_3_std + value: 17.372799999999998 + - type: nauc_recall_at_3_diff1 + value: 63.488 + - type: nauc_recall_at_5_max + value: 65.4804 + - type: nauc_recall_at_5_std + value: 28.376 + - type: nauc_recall_at_5_diff1 + value: 62.4274 + - type: nauc_recall_at_10_max + value: 67.7459 + - type: nauc_recall_at_10_std + value: 40.8339 + - type: nauc_recall_at_10_diff1 + value: 59.2704 + - type: nauc_recall_at_20_max + value: 67.4241 + - type: nauc_recall_at_20_std + value: 49.1244 + - type: nauc_recall_at_20_diff1 + value: 57.3728 + - type: nauc_recall_at_100_max + value: 71.1514 + - type: nauc_recall_at_100_std + value: 71.35510000000001 + - type: nauc_recall_at_100_diff1 + value: 55.964800000000004 + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_precision_at_1_max + value: 54.2597 + - type: nauc_precision_at_1_std + value: 9.9915 + - type: nauc_precision_at_1_diff1 + value: 72.0029 + - type: nauc_precision_at_3_max + value: 62.5301 + - type: nauc_precision_at_3_std + value: 17.372799999999998 + - type: nauc_precision_at_3_diff1 + value: 63.488 + - type: nauc_precision_at_5_max + value: 65.4804 + - type: nauc_precision_at_5_std + value: 28.376 + - type: nauc_precision_at_5_diff1 + value: 62.4274 + - type: nauc_precision_at_10_max + value: 67.7459 + - type: nauc_precision_at_10_std + value: 40.8339 + - type: nauc_precision_at_10_diff1 + value: 59.2704 + - type: nauc_precision_at_20_max + value: 67.4241 + - type: nauc_precision_at_20_std + value: 49.1244 + - type: nauc_precision_at_20_diff1 + value: 57.3728 + - type: nauc_precision_at_100_max + value: 71.1514 + - type: nauc_precision_at_100_std + value: 71.35510000000001 + - type: nauc_precision_at_100_diff1 + value: 55.964800000000004 + - type: nauc_precision_at_1000_max + value: .nan + - type: nauc_precision_at_1000_std + value: .nan + - type: nauc_precision_at_1000_diff1 + value: .nan + - type: nauc_mrr_at_1_max + value: 54.2597 + - type: nauc_mrr_at_1_std + value: 9.9915 + - type: nauc_mrr_at_1_diff1 + value: 72.0029 + - type: nauc_mrr_at_3_max + value: 57.4277 + - type: nauc_mrr_at_3_std + value: 12.1778 + - type: nauc_mrr_at_3_diff1 + value: 69.0312 + - type: nauc_mrr_at_5_max + value: 57.7291 + - type: nauc_mrr_at_5_std + value: 13.655800000000001 + - type: nauc_mrr_at_5_diff1 + value: 69.0376 + - type: nauc_mrr_at_10_max + value: 57.7091 + - type: nauc_mrr_at_10_std + value: 14.2236 + - type: nauc_mrr_at_10_diff1 + value: 68.99849999999999 + - type: nauc_mrr_at_20_max + value: 57.605700000000006 + - type: nauc_mrr_at_20_std + value: 14.2305 + - type: nauc_mrr_at_20_diff1 + value: 69.0304 + - type: nauc_mrr_at_100_max + value: 57.6007 + - type: nauc_mrr_at_100_std + value: 14.219499999999998 + - type: nauc_mrr_at_100_diff1 + value: 69.0682 + - type: nauc_mrr_at_1000_max + value: 57.5939 + - type: nauc_mrr_at_1000_std + value: 14.1793 + - type: nauc_mrr_at_1000_diff1 + value: 69.0767 + - type: main_score + value: 74.736 + task: + type: Retrieval + - dataset: + config: java + name: MTEB CodeSearchNetRetrieval (java) + revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 + split: test + type: code-search-net/code_search_net + metrics: + - type: ndcg_at_1 + value: 55.1 + - type: ndcg_at_3 + value: 66.89399999999999 + - type: ndcg_at_5 + value: 68.89999999999999 + - type: ndcg_at_10 + value: 70.89 + - type: ndcg_at_20 + value: 72.016 + - type: ndcg_at_100 + value: 73.047 + - type: ndcg_at_1000 + value: 73.553 + - type: map_at_1 + value: 55.1 + - type: map_at_3 + value: 64.05 + - type: map_at_5 + value: 65.18 + - type: map_at_10 + value: 66.012 + - type: map_at_20 + value: 66.328 + - type: map_at_100 + value: 66.483 + - type: map_at_1000 + value: 66.498 + - type: recall_at_1 + value: 55.1 + - type: recall_at_3 + value: 75.1 + - type: recall_at_5 + value: 79.9 + - type: recall_at_10 + value: 86.0 + - type: recall_at_20 + value: 90.4 + - type: recall_at_100 + value: 95.8 + - type: recall_at_1000 + value: 100.0 + - type: precision_at_1 + value: 55.1 + - type: precision_at_3 + value: 25.033 + - type: precision_at_5 + value: 15.98 + - type: precision_at_10 + value: 8.6 + - type: precision_at_20 + value: 4.52 + - type: precision_at_100 + value: 0.958 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 55.1 + - type: mrr_at_3 + value: 64.05 + - type: mrr_at_5 + value: 65.18 + - type: mrr_at_10 + value: 66.0123 + - type: mrr_at_20 + value: 66.32820000000001 + - type: mrr_at_100 + value: 66.4827 + - type: mrr_at_1000 + value: 66.49810000000001 + - type: nauc_ndcg_at_1_max + value: 30.206100000000003 + - type: nauc_ndcg_at_1_std + value: -14.6389 + - type: nauc_ndcg_at_1_diff1 + value: 61.8849 + - type: nauc_ndcg_at_3_max + value: 32.7259 + - type: nauc_ndcg_at_3_std + value: -11.568399999999999 + - type: nauc_ndcg_at_3_diff1 + value: 59.918800000000005 + - type: nauc_ndcg_at_5_max + value: 34.1822 + - type: nauc_ndcg_at_5_std + value: -8.104 + - type: nauc_ndcg_at_5_diff1 + value: 59.434799999999996 + - type: nauc_ndcg_at_10_max + value: 36.1247 + - type: nauc_ndcg_at_10_std + value: -6.585100000000001 + - type: nauc_ndcg_at_10_diff1 + value: 59.2885 + - type: nauc_ndcg_at_20_max + value: 35.9396 + - type: nauc_ndcg_at_20_std + value: -6.0885 + - type: nauc_ndcg_at_20_diff1 + value: 59.4417 + - type: nauc_ndcg_at_100_max + value: 35.951499999999996 + - type: nauc_ndcg_at_100_std + value: -6.1491 + - type: nauc_ndcg_at_100_diff1 + value: 60.3437 + - type: nauc_ndcg_at_1000_max + value: 34.7092 + - type: nauc_ndcg_at_1000_std + value: -8.0607 + - type: nauc_ndcg_at_1000_diff1 + value: 60.0215 + - type: nauc_map_at_1_max + value: 30.206100000000003 + - type: nauc_map_at_1_std + value: -14.6389 + - type: nauc_map_at_1_diff1 + value: 61.8849 + - type: nauc_map_at_3_max + value: 31.9303 + - type: nauc_map_at_3_std + value: -12.651200000000001 + - type: nauc_map_at_3_diff1 + value: 60.33 + - type: nauc_map_at_5_max + value: 32.6537 + - type: nauc_map_at_5_std + value: -10.8746 + - type: nauc_map_at_5_diff1 + value: 60.0754 + - type: nauc_map_at_10_max + value: 33.269 + - type: nauc_map_at_10_std + value: -10.4054 + - type: nauc_map_at_10_diff1 + value: 60.0235 + - type: nauc_map_at_20_max + value: 33.1875 + - type: nauc_map_at_20_std + value: -10.3417 + - type: nauc_map_at_20_diff1 + value: 60.067899999999995 + - type: nauc_map_at_100_max + value: 33.213 + - type: nauc_map_at_100_std + value: -10.3299 + - type: nauc_map_at_100_diff1 + value: 60.166399999999996 + - type: nauc_map_at_1000_max + value: 33.186 + - type: nauc_map_at_1000_std + value: -10.3713 + - type: nauc_map_at_1000_diff1 + value: 60.16010000000001 + - type: nauc_recall_at_1_max + value: 30.206100000000003 + - type: nauc_recall_at_1_std + value: -14.6389 + - type: nauc_recall_at_1_diff1 + value: 61.8849 + - type: nauc_recall_at_3_max + value: 35.7096 + - type: nauc_recall_at_3_std + value: -7.4548000000000005 + - type: nauc_recall_at_3_diff1 + value: 58.475699999999996 + - type: nauc_recall_at_5_max + value: 41.0231 + - type: nauc_recall_at_5_std + value: 4.4421 + - type: nauc_recall_at_5_diff1 + value: 56.7391 + - type: nauc_recall_at_10_max + value: 54.789 + - type: nauc_recall_at_10_std + value: 17.7044 + - type: nauc_recall_at_10_diff1 + value: 55.0592 + - type: nauc_recall_at_20_max + value: 60.7809 + - type: nauc_recall_at_20_std + value: 32.4021 + - type: nauc_recall_at_20_diff1 + value: 54.7663 + - type: nauc_recall_at_100_max + value: 89.4591 + - type: nauc_recall_at_100_std + value: 76.2783 + - type: nauc_recall_at_100_diff1 + value: 74.4576 + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_precision_at_1_max + value: 30.206100000000003 + - type: nauc_precision_at_1_std + value: -14.6389 + - type: nauc_precision_at_1_diff1 + value: 61.8849 + - type: nauc_precision_at_3_max + value: 35.7096 + - type: nauc_precision_at_3_std + value: -7.4548000000000005 + - type: nauc_precision_at_3_diff1 + value: 58.475699999999996 + - type: nauc_precision_at_5_max + value: 41.0231 + - type: nauc_precision_at_5_std + value: 4.4421 + - type: nauc_precision_at_5_diff1 + value: 56.7391 + - type: nauc_precision_at_10_max + value: 54.789 + - type: nauc_precision_at_10_std + value: 17.7044 + - type: nauc_precision_at_10_diff1 + value: 55.0592 + - type: nauc_precision_at_20_max + value: 60.7809 + - type: nauc_precision_at_20_std + value: 32.4021 + - type: nauc_precision_at_20_diff1 + value: 54.7663 + - type: nauc_precision_at_100_max + value: 89.4591 + - type: nauc_precision_at_100_std + value: 76.2783 + - type: nauc_precision_at_100_diff1 + value: 74.4576 + - type: nauc_precision_at_1000_max + value: .nan + - type: nauc_precision_at_1000_std + value: .nan + - type: nauc_precision_at_1000_diff1 + value: .nan + - type: nauc_mrr_at_1_max + value: 30.206100000000003 + - type: nauc_mrr_at_1_std + value: -14.6389 + - type: nauc_mrr_at_1_diff1 + value: 61.8849 + - type: nauc_mrr_at_3_max + value: 31.9303 + - type: nauc_mrr_at_3_std + value: -12.651200000000001 + - type: nauc_mrr_at_3_diff1 + value: 60.33 + - type: nauc_mrr_at_5_max + value: 32.6537 + - type: nauc_mrr_at_5_std + value: -10.8746 + - type: nauc_mrr_at_5_diff1 + value: 60.0754 + - type: nauc_mrr_at_10_max + value: 33.269 + - type: nauc_mrr_at_10_std + value: -10.4054 + - type: nauc_mrr_at_10_diff1 + value: 60.0235 + - type: nauc_mrr_at_20_max + value: 33.1875 + - type: nauc_mrr_at_20_std + value: -10.3417 + - type: nauc_mrr_at_20_diff1 + value: 60.067899999999995 + - type: nauc_mrr_at_100_max + value: 33.213 + - type: nauc_mrr_at_100_std + value: -10.3299 + - type: nauc_mrr_at_100_diff1 + value: 60.166399999999996 + - type: nauc_mrr_at_1000_max + value: 33.186 + - type: nauc_mrr_at_1000_std + value: -10.3713 + - type: nauc_mrr_at_1000_diff1 + value: 60.16010000000001 + - type: main_score + value: 70.89 + task: + type: Retrieval + - dataset: + config: php + name: MTEB CodeSearchNetRetrieval (php) + revision: fdc6a9e39575768c27eb8a2a5f702bf846eb4759 + split: test + type: code-search-net/code_search_net + metrics: + - type: ndcg_at_1 + value: 56.89999999999999 + - type: ndcg_at_3 + value: 69.128 + - type: ndcg_at_5 + value: 71.495 + - type: ndcg_at_10 + value: 72.92999999999999 + - type: ndcg_at_20 + value: 73.775 + - type: ndcg_at_100 + value: 74.476 + - type: ndcg_at_1000 + value: 75.075 + - type: map_at_1 + value: 56.89999999999999 + - type: map_at_3 + value: 66.10000000000001 + - type: map_at_5 + value: 67.425 + - type: map_at_10 + value: 68.024 + - type: map_at_20 + value: 68.26100000000001 + - type: map_at_100 + value: 68.357 + - type: map_at_1000 + value: 68.376 + - type: recall_at_1 + value: 56.89999999999999 + - type: recall_at_3 + value: 77.9 + - type: recall_at_5 + value: 83.6 + - type: recall_at_10 + value: 88.0 + - type: recall_at_20 + value: 91.3 + - type: recall_at_100 + value: 95.1 + - type: recall_at_1000 + value: 100.0 + - type: precision_at_1 + value: 56.89999999999999 + - type: precision_at_3 + value: 25.967000000000002 + - type: precision_at_5 + value: 16.72 + - type: precision_at_10 + value: 8.799999999999999 + - type: precision_at_20 + value: 4.565 + - type: precision_at_100 + value: 0.951 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 56.89999999999999 + - type: mrr_at_3 + value: 66.10000000000001 + - type: mrr_at_5 + value: 67.425 + - type: mrr_at_10 + value: 68.0238 + - type: mrr_at_20 + value: 68.2613 + - type: mrr_at_100 + value: 68.35719999999999 + - type: mrr_at_1000 + value: 68.3763 + - type: nauc_ndcg_at_1_max + value: 43.5297 + - type: nauc_ndcg_at_1_std + value: 7.986600000000001 + - type: nauc_ndcg_at_1_diff1 + value: 65.95689999999999 + - type: nauc_ndcg_at_3_max + value: 52.166500000000006 + - type: nauc_ndcg_at_3_std + value: 17.0778 + - type: nauc_ndcg_at_3_diff1 + value: 60.8598 + - type: nauc_ndcg_at_5_max + value: 53.1733 + - type: nauc_ndcg_at_5_std + value: 18.7316 + - type: nauc_ndcg_at_5_diff1 + value: 61.4908 + - type: nauc_ndcg_at_10_max + value: 53.6245 + - type: nauc_ndcg_at_10_std + value: 19.5627 + - type: nauc_ndcg_at_10_diff1 + value: 61.9788 + - type: nauc_ndcg_at_20_max + value: 53.725199999999994 + - type: nauc_ndcg_at_20_std + value: 20.5901 + - type: nauc_ndcg_at_20_diff1 + value: 62.480199999999996 + - type: nauc_ndcg_at_100_max + value: 53.083499999999994 + - type: nauc_ndcg_at_100_std + value: 19.8779 + - type: nauc_ndcg_at_100_diff1 + value: 62.849 + - type: nauc_ndcg_at_1000_max + value: 51.9568 + - type: nauc_ndcg_at_1000_std + value: 17.8629 + - type: nauc_ndcg_at_1000_diff1 + value: 62.7251 + - type: nauc_map_at_1_max + value: 43.5297 + - type: nauc_map_at_1_std + value: 7.986600000000001 + - type: nauc_map_at_1_diff1 + value: 65.95689999999999 + - type: nauc_map_at_3_max + value: 49.7136 + - type: nauc_map_at_3_std + value: 14.054400000000001 + - type: nauc_map_at_3_diff1 + value: 62.3127 + - type: nauc_map_at_5_max + value: 50.138400000000004 + - type: nauc_map_at_5_std + value: 14.7824 + - type: nauc_map_at_5_diff1 + value: 62.6784 + - type: nauc_map_at_10_max + value: 50.2613 + - type: nauc_map_at_10_std + value: 15.024899999999999 + - type: nauc_map_at_10_diff1 + value: 62.864200000000004 + - type: nauc_map_at_20_max + value: 50.267300000000006 + - type: nauc_map_at_20_std + value: 15.234300000000001 + - type: nauc_map_at_20_diff1 + value: 63.00130000000001 + - type: nauc_map_at_100_max + value: 50.1927 + - type: nauc_map_at_100_std + value: 15.1701 + - type: nauc_map_at_100_diff1 + value: 63.0549 + - type: nauc_map_at_1000_max + value: 50.1623 + - type: nauc_map_at_1000_std + value: 15.118500000000001 + - type: nauc_map_at_1000_diff1 + value: 63.048300000000005 + - type: nauc_recall_at_1_max + value: 43.5297 + - type: nauc_recall_at_1_std + value: 7.986600000000001 + - type: nauc_recall_at_1_diff1 + value: 65.95689999999999 + - type: nauc_recall_at_3_max + value: 61.7214 + - type: nauc_recall_at_3_std + value: 29.1046 + - type: nauc_recall_at_3_diff1 + value: 55.1971 + - type: nauc_recall_at_5_max + value: 68.1151 + - type: nauc_recall_at_5_std + value: 38.587700000000005 + - type: nauc_recall_at_5_diff1 + value: 55.886 + - type: nauc_recall_at_10_max + value: 75.3834 + - type: nauc_recall_at_10_std + value: 49.6516 + - type: nauc_recall_at_10_diff1 + value: 57.0852 + - type: nauc_recall_at_20_max + value: 83.7342 + - type: nauc_recall_at_20_std + value: 69.9947 + - type: nauc_recall_at_20_diff1 + value: 60.002500000000005 + - type: nauc_recall_at_100_max + value: 91.4204 + - type: nauc_recall_at_100_std + value: 89.0309 + - type: nauc_recall_at_100_diff1 + value: 65.7358 + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_precision_at_1_max + value: 43.5297 + - type: nauc_precision_at_1_std + value: 7.986600000000001 + - type: nauc_precision_at_1_diff1 + value: 65.95689999999999 + - type: nauc_precision_at_3_max + value: 61.7214 + - type: nauc_precision_at_3_std + value: 29.1046 + - type: nauc_precision_at_3_diff1 + value: 55.1971 + - type: nauc_precision_at_5_max + value: 68.1151 + - type: nauc_precision_at_5_std + value: 38.587700000000005 + - type: nauc_precision_at_5_diff1 + value: 55.886 + - type: nauc_precision_at_10_max + value: 75.3834 + - type: nauc_precision_at_10_std + value: 49.6516 + - type: nauc_precision_at_10_diff1 + value: 57.0852 + - type: nauc_precision_at_20_max + value: 83.7342 + - type: nauc_precision_at_20_std + value: 69.9947 + - type: nauc_precision_at_20_diff1 + value: 60.002500000000005 + - type: nauc_precision_at_100_max + value: 91.4204 + - type: nauc_precision_at_100_std + value: 89.0309 + - type: nauc_precision_at_100_diff1 + value: 65.7358 + - type: nauc_precision_at_1000_max + value: .nan + - type: nauc_precision_at_1000_std + value: .nan + - type: nauc_precision_at_1000_diff1 + value: .nan + - type: nauc_mrr_at_1_max + value: 43.5297 + - type: nauc_mrr_at_1_std + value: 7.986600000000001 + - type: nauc_mrr_at_1_diff1 + value: 65.95689999999999 + - type: nauc_mrr_at_3_max + value: 49.7136 + - type: nauc_mrr_at_3_std + value: 14.054400000000001 + - type: nauc_mrr_at_3_diff1 + value: 62.3127 + - type: nauc_mrr_at_5_max + value: 50.138400000000004 + - type: nauc_mrr_at_5_std + value: 14.7824 + - type: nauc_mrr_at_5_diff1 + value: 62.6784 + - type: nauc_mrr_at_10_max + value: 50.2613 + - type: nauc_mrr_at_10_std + value: 15.024899999999999 + - type: nauc_mrr_at_10_diff1 + value: 62.864200000000004 + - type: nauc_mrr_at_20_max + value: 50.267300000000006 + - type: nauc_mrr_at_20_std + value: 15.234300000000001 + - type: nauc_mrr_at_20_diff1 + value: 63.00130000000001 + - type: nauc_mrr_at_100_max + value: 50.1927 + - type: nauc_mrr_at_100_std + value: 15.1701 + - type: nauc_mrr_at_100_diff1 + value: 63.0549 + - type: nauc_mrr_at_1000_max + value: 50.1623 + - type: nauc_mrr_at_1000_std + value: 15.118500000000001 + - type: nauc_mrr_at_1000_diff1 + value: 63.048300000000005 + - type: main_score + value: 72.92999999999999 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CodeTransOceanContest (default) + revision: 20da4eb20a4b17300c0986ee148c90867a7f2a4d + split: test + type: CoIR-Retrieval/codetrans-contest + metrics: + - type: ndcg_at_1 + value: 50.226000000000006 + - type: ndcg_at_3 + value: 55.748 + - type: ndcg_at_5 + value: 58.007 + - type: ndcg_at_10 + value: 60.831 + - type: ndcg_at_20 + value: 62.793 + - type: ndcg_at_100 + value: 64.43299999999999 + - type: ndcg_at_1000 + value: 65.60000000000001 + - type: map_at_1 + value: 50.226000000000006 + - type: map_at_3 + value: 54.374 + - type: map_at_5 + value: 55.641 + - type: map_at_10 + value: 56.83200000000001 + - type: map_at_20 + value: 57.379999999999995 + - type: map_at_100 + value: 57.594 + - type: map_at_1000 + value: 57.633 + - type: recall_at_1 + value: 50.226000000000006 + - type: recall_at_3 + value: 59.729 + - type: recall_at_5 + value: 65.158 + - type: recall_at_10 + value: 73.756 + - type: recall_at_20 + value: 81.448 + - type: recall_at_100 + value: 90.498 + - type: recall_at_1000 + value: 100.0 + - type: precision_at_1 + value: 50.226000000000006 + - type: precision_at_3 + value: 19.91 + - type: precision_at_5 + value: 13.032 + - type: precision_at_10 + value: 7.376 + - type: precision_at_20 + value: 4.072 + - type: precision_at_100 + value: 0.905 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 50.2262 + - type: mrr_at_3 + value: 54.374100000000006 + - type: mrr_at_5 + value: 55.641 + - type: mrr_at_10 + value: 56.8322 + - type: mrr_at_20 + value: 57.3798 + - type: mrr_at_100 + value: 57.594500000000004 + - type: mrr_at_1000 + value: 57.6333 + - type: nauc_ndcg_at_1_max + value: 57.24249999999999 + - type: nauc_ndcg_at_1_std + value: 3.4893 + - type: nauc_ndcg_at_1_diff1 + value: 74.5093 + - type: nauc_ndcg_at_3_max + value: 57.099 + - type: nauc_ndcg_at_3_std + value: 3.3562000000000003 + - type: nauc_ndcg_at_3_diff1 + value: 71.5239 + - type: nauc_ndcg_at_5_max + value: 57.5998 + - type: nauc_ndcg_at_5_std + value: 4.7879 + - type: nauc_ndcg_at_5_diff1 + value: 69.9839 + - type: nauc_ndcg_at_10_max + value: 56.1631 + - type: nauc_ndcg_at_10_std + value: 6.0869 + - type: nauc_ndcg_at_10_diff1 + value: 68.32939999999999 + - type: nauc_ndcg_at_20_max + value: 56.098800000000004 + - type: nauc_ndcg_at_20_std + value: 5.1246 + - type: nauc_ndcg_at_20_diff1 + value: 68.9858 + - type: nauc_ndcg_at_100_max + value: 56.788799999999995 + - type: nauc_ndcg_at_100_std + value: 5.6714 + - type: nauc_ndcg_at_100_diff1 + value: 69.3668 + - type: nauc_ndcg_at_1000_max + value: 56.7396 + - type: nauc_ndcg_at_1000_std + value: 5.0106 + - type: nauc_ndcg_at_1000_diff1 + value: 70.1024 + - type: nauc_map_at_1_max + value: 57.24249999999999 + - type: nauc_map_at_1_std + value: 3.4893 + - type: nauc_map_at_1_diff1 + value: 74.5093 + - type: nauc_map_at_3_max + value: 57.2832 + - type: nauc_map_at_3_std + value: 3.4703999999999997 + - type: nauc_map_at_3_diff1 + value: 72.40490000000001 + - type: nauc_map_at_5_max + value: 57.5445 + - type: nauc_map_at_5_std + value: 4.1418 + - type: nauc_map_at_5_diff1 + value: 71.5756 + - type: nauc_map_at_10_max + value: 57.0669 + - type: nauc_map_at_10_std + value: 4.7488 + - type: nauc_map_at_10_diff1 + value: 70.97869999999999 + - type: nauc_map_at_20_max + value: 57.08800000000001 + - type: nauc_map_at_20_std + value: 4.4653 + - type: nauc_map_at_20_diff1 + value: 71.2187 + - type: nauc_map_at_100_max + value: 57.1484 + - type: nauc_map_at_100_std + value: 4.5175 + - type: nauc_map_at_100_diff1 + value: 71.2734 + - type: nauc_map_at_1000_max + value: 57.1356 + - type: nauc_map_at_1000_std + value: 4.4929 + - type: nauc_map_at_1000_diff1 + value: 71.28710000000001 + - type: nauc_recall_at_1_max + value: 57.24249999999999 + - type: nauc_recall_at_1_std + value: 3.4893 + - type: nauc_recall_at_1_diff1 + value: 74.5093 + - type: nauc_recall_at_3_max + value: 56.469800000000006 + - type: nauc_recall_at_3_std + value: 2.9709 + - type: nauc_recall_at_3_diff1 + value: 68.7698 + - type: nauc_recall_at_5_max + value: 57.811 + - type: nauc_recall_at_5_std + value: 7.2669999999999995 + - type: nauc_recall_at_5_diff1 + value: 64.4325 + - type: nauc_recall_at_10_max + value: 51.5712 + - type: nauc_recall_at_10_std + value: 12.1867 + - type: nauc_recall_at_10_diff1 + value: 56.4929 + - type: nauc_recall_at_20_max + value: 49.3 + - type: nauc_recall_at_20_std + value: 8.371599999999999 + - type: nauc_recall_at_20_diff1 + value: 56.2505 + - type: nauc_recall_at_100_max + value: 55.7663 + - type: nauc_recall_at_100_std + value: 19.9214 + - type: nauc_recall_at_100_diff1 + value: 51.6979 + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_precision_at_1_max + value: 57.24249999999999 + - type: nauc_precision_at_1_std + value: 3.4893 + - type: nauc_precision_at_1_diff1 + value: 74.5093 + - type: nauc_precision_at_3_max + value: 56.469800000000006 + - type: nauc_precision_at_3_std + value: 2.9709 + - type: nauc_precision_at_3_diff1 + value: 68.7698 + - type: nauc_precision_at_5_max + value: 57.811 + - type: nauc_precision_at_5_std + value: 7.2669999999999995 + - type: nauc_precision_at_5_diff1 + value: 64.4325 + - type: nauc_precision_at_10_max + value: 51.5712 + - type: nauc_precision_at_10_std + value: 12.1867 + - type: nauc_precision_at_10_diff1 + value: 56.4929 + - type: nauc_precision_at_20_max + value: 49.3 + - type: nauc_precision_at_20_std + value: 8.371599999999999 + - type: nauc_precision_at_20_diff1 + value: 56.2505 + - type: nauc_precision_at_100_max + value: 55.7663 + - type: nauc_precision_at_100_std + value: 19.9214 + - type: nauc_precision_at_100_diff1 + value: 51.6979 + - type: nauc_precision_at_1000_max + value: 100.0 + - type: nauc_precision_at_1000_std + value: 100.0 + - type: nauc_precision_at_1000_diff1 + value: 100.0 + - type: nauc_mrr_at_1_max + value: 57.24249999999999 + - type: nauc_mrr_at_1_std + value: 3.4893 + - type: nauc_mrr_at_1_diff1 + value: 74.5093 + - type: nauc_mrr_at_3_max + value: 57.2832 + - type: nauc_mrr_at_3_std + value: 3.4703999999999997 + - type: nauc_mrr_at_3_diff1 + value: 72.40490000000001 + - type: nauc_mrr_at_5_max + value: 57.5445 + - type: nauc_mrr_at_5_std + value: 4.1418 + - type: nauc_mrr_at_5_diff1 + value: 71.5756 + - type: nauc_mrr_at_10_max + value: 57.0669 + - type: nauc_mrr_at_10_std + value: 4.7488 + - type: nauc_mrr_at_10_diff1 + value: 70.97869999999999 + - type: nauc_mrr_at_20_max + value: 57.08800000000001 + - type: nauc_mrr_at_20_std + value: 4.4653 + - type: nauc_mrr_at_20_diff1 + value: 71.2187 + - type: nauc_mrr_at_100_max + value: 57.1484 + - type: nauc_mrr_at_100_std + value: 4.5175 + - type: nauc_mrr_at_100_diff1 + value: 71.2734 + - type: nauc_mrr_at_1000_max + value: 57.1356 + - type: nauc_mrr_at_1000_std + value: 4.4929 + - type: nauc_mrr_at_1000_diff1 + value: 71.28710000000001 + - type: main_score + value: 60.831 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CodeTransOceanDL (default) + revision: 281562cb8a1265ab5c0824bfa6ddcd9b0a15618f + split: test + type: CoIR-Retrieval/codetrans-dl + metrics: + - type: ndcg_at_1 + value: 8.889 + - type: ndcg_at_3 + value: 12.09 + - type: ndcg_at_5 + value: 18.355 + - type: ndcg_at_10 + value: 32.138 + - type: ndcg_at_20 + value: 38.437 + - type: ndcg_at_100 + value: 39.031 + - type: ndcg_at_1000 + value: 39.031 + - type: map_at_1 + value: 8.889 + - type: map_at_3 + value: 11.111 + - type: map_at_5 + value: 14.639 + - type: map_at_10 + value: 20.193 + - type: map_at_20 + value: 22.137 + - type: map_at_100 + value: 22.21 + - type: map_at_1000 + value: 22.21 + - type: recall_at_1 + value: 8.889 + - type: recall_at_3 + value: 15.0 + - type: recall_at_5 + value: 30.0 + - type: recall_at_10 + value: 73.333 + - type: recall_at_20 + value: 96.667 + - type: recall_at_100 + value: 100.0 + - type: recall_at_1000 + value: 100.0 + - type: precision_at_1 + value: 8.889 + - type: precision_at_3 + value: 5.0 + - type: precision_at_5 + value: 6.0 + - type: precision_at_10 + value: 7.333 + - type: precision_at_20 + value: 4.833 + - type: precision_at_100 + value: 1.0 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 6.1110999999999995 + - type: mrr_at_3 + value: 10.0 + - type: mrr_at_5 + value: 12.8056 + - type: mrr_at_10 + value: 19.164900000000003 + - type: mrr_at_20 + value: 20.8374 + - type: mrr_at_100 + value: 20.9115 + - type: mrr_at_1000 + value: 20.9115 + - type: nauc_ndcg_at_1_max + value: -40.8791 + - type: nauc_ndcg_at_1_std + value: -29.137 + - type: nauc_ndcg_at_1_diff1 + value: -25.7462 + - type: nauc_ndcg_at_3_max + value: -43.8611 + - type: nauc_ndcg_at_3_std + value: -31.619999999999997 + - type: nauc_ndcg_at_3_diff1 + value: -8.387799999999999 + - type: nauc_ndcg_at_5_max + value: -34.1018 + - type: nauc_ndcg_at_5_std + value: -20.9725 + - type: nauc_ndcg_at_5_diff1 + value: -14.6478 + - type: nauc_ndcg_at_10_max + value: -29.694599999999998 + - type: nauc_ndcg_at_10_std + value: -17.6602 + - type: nauc_ndcg_at_10_diff1 + value: -21.0388 + - type: nauc_ndcg_at_20_max + value: -42.308800000000005 + - type: nauc_ndcg_at_20_std + value: -20.778 + - type: nauc_ndcg_at_20_diff1 + value: -15.67 + - type: nauc_ndcg_at_100_max + value: -37.4946 + - type: nauc_ndcg_at_100_std + value: -22.2861 + - type: nauc_ndcg_at_100_diff1 + value: -16.020300000000002 + - type: nauc_ndcg_at_1000_max + value: -37.4946 + - type: nauc_ndcg_at_1000_std + value: -22.2861 + - type: nauc_ndcg_at_1000_diff1 + value: -16.020300000000002 + - type: nauc_map_at_1_max + value: -40.8791 + - type: nauc_map_at_1_std + value: -29.137 + - type: nauc_map_at_1_diff1 + value: -25.7462 + - type: nauc_map_at_3_max + value: -43.1058 + - type: nauc_map_at_3_std + value: -31.071900000000003 + - type: nauc_map_at_3_diff1 + value: -12.875900000000001 + - type: nauc_map_at_5_max + value: -36.4737 + - type: nauc_map_at_5_std + value: -23.8979 + - type: nauc_map_at_5_diff1 + value: -16.206400000000002 + - type: nauc_map_at_10_max + value: -34.2318 + - type: nauc_map_at_10_std + value: -22.0811 + - type: nauc_map_at_10_diff1 + value: -18.5454 + - type: nauc_map_at_20_max + value: -37.9204 + - type: nauc_map_at_20_std + value: -23.3876 + - type: nauc_map_at_20_diff1 + value: -16.8628 + - type: nauc_map_at_100_max + value: -37.401 + - type: nauc_map_at_100_std + value: -23.595299999999998 + - type: nauc_map_at_100_diff1 + value: -16.8443 + - type: nauc_map_at_1000_max + value: -37.401 + - type: nauc_map_at_1000_std + value: -23.595299999999998 + - type: nauc_map_at_1000_diff1 + value: -16.8443 + - type: nauc_recall_at_1_max + value: -40.8791 + - type: nauc_recall_at_1_std + value: -29.137 + - type: nauc_recall_at_1_diff1 + value: -25.7462 + - type: nauc_recall_at_3_max + value: -45.6372 + - type: nauc_recall_at_3_std + value: -32.8876 + - type: nauc_recall_at_3_diff1 + value: 2.1906 + - type: nauc_recall_at_5_max + value: -29.531299999999998 + - type: nauc_recall_at_5_std + value: -15.2907 + - type: nauc_recall_at_5_diff1 + value: -12.279900000000001 + - type: nauc_recall_at_10_max + value: -17.0981 + - type: nauc_recall_at_10_std + value: -5.6821 + - type: nauc_recall_at_10_diff1 + value: -31.382700000000003 + - type: nauc_recall_at_20_max + value: -164.1923 + - type: nauc_recall_at_20_std + value: 14.6592 + - type: nauc_recall_at_20_diff1 + value: -1.6729 + - type: nauc_recall_at_100_max + value: .nan + - type: nauc_recall_at_100_std + value: .nan + - type: nauc_recall_at_100_diff1 + value: .nan + - type: nauc_recall_at_1000_max + value: .nan + - type: nauc_recall_at_1000_std + value: .nan + - type: nauc_recall_at_1000_diff1 + value: .nan + - type: nauc_precision_at_1_max + value: -40.8791 + - type: nauc_precision_at_1_std + value: -29.137 + - type: nauc_precision_at_1_diff1 + value: -25.7462 + - type: nauc_precision_at_3_max + value: -45.6372 + - type: nauc_precision_at_3_std + value: -32.8876 + - type: nauc_precision_at_3_diff1 + value: 2.1906 + - type: nauc_precision_at_5_max + value: -29.531299999999998 + - type: nauc_precision_at_5_std + value: -15.2907 + - type: nauc_precision_at_5_diff1 + value: -12.279900000000001 + - type: nauc_precision_at_10_max + value: -17.0981 + - type: nauc_precision_at_10_std + value: -5.6821 + - type: nauc_precision_at_10_diff1 + value: -31.382700000000003 + - type: nauc_precision_at_20_max + value: -164.1923 + - type: nauc_precision_at_20_std + value: 14.6592 + - type: nauc_precision_at_20_diff1 + value: -1.6729 + - type: nauc_precision_at_100_max + value: 100.0 + - type: nauc_precision_at_100_std + value: 100.0 + - type: nauc_precision_at_100_diff1 + value: 100.0 + - type: nauc_precision_at_1000_max + value: 100.0 + - type: nauc_precision_at_1000_std + value: 100.0 + - type: nauc_precision_at_1000_diff1 + value: 100.0 + - type: nauc_mrr_at_1_max + value: -38.4833 + - type: nauc_mrr_at_1_std + value: -27.4288 + - type: nauc_mrr_at_1_diff1 + value: -2.3441 + - type: nauc_mrr_at_3_max + value: -40.2427 + - type: nauc_mrr_at_3_std + value: -28.479 + - type: nauc_mrr_at_3_diff1 + value: 14.5837 + - type: nauc_mrr_at_5_max + value: -32.784400000000005 + - type: nauc_mrr_at_5_std + value: -19.3984 + - type: nauc_mrr_at_5_diff1 + value: 8.2762 + - type: nauc_mrr_at_10_max + value: -31.999499999999998 + - type: nauc_mrr_at_10_std + value: -20.9878 + - type: nauc_mrr_at_10_diff1 + value: 9.2346 + - type: nauc_mrr_at_20_max + value: -36.2588 + - type: nauc_mrr_at_20_std + value: -21.057699999999997 + - type: nauc_mrr_at_20_diff1 + value: 9.4499 + - type: nauc_mrr_at_100_max + value: -35.6528 + - type: nauc_mrr_at_100_std + value: -21.288 + - type: nauc_mrr_at_100_diff1 + value: 9.591 + - type: nauc_mrr_at_1000_max + value: -35.6528 + - type: nauc_mrr_at_1000_std + value: -21.288 + - type: nauc_mrr_at_1000_diff1 + value: 9.591 + - type: main_score + value: 32.138 + task: + type: Retrieval + - dataset: + config: default + name: MTEB CosQA (default) + revision: bc5efb7e9d437246ce393ed19d772e08e4a79535 + split: test + type: CoIR-Retrieval/cosqa + metrics: + - type: ndcg_at_1 + value: 14.6 + - type: ndcg_at_3 + value: 23.043 + - type: ndcg_at_5 + value: 28.551 + - type: ndcg_at_10 + value: 33.452 + - type: ndcg_at_20 + value: 37.094 + - type: ndcg_at_100 + value: 40.416999999999994 + - type: ndcg_at_1000 + value: 41.684 + - type: map_at_1 + value: 14.6 + - type: map_at_3 + value: 20.8 + - type: map_at_5 + value: 23.849999999999998 + - type: map_at_10 + value: 25.941 + - type: map_at_20 + value: 26.941 + - type: map_at_100 + value: 27.418 + - type: map_at_1000 + value: 27.473999999999997 + - type: recall_at_1 + value: 14.6 + - type: recall_at_3 + value: 29.599999999999998 + - type: recall_at_5 + value: 43.0 + - type: recall_at_10 + value: 57.8 + - type: recall_at_20 + value: 72.2 + - type: recall_at_100 + value: 89.8 + - type: recall_at_1000 + value: 99.4 + - type: precision_at_1 + value: 14.6 + - type: precision_at_3 + value: 9.866999999999999 + - type: precision_at_5 + value: 8.6 + - type: precision_at_10 + value: 5.779999999999999 + - type: precision_at_20 + value: 3.61 + - type: precision_at_100 + value: 0.898 + - type: precision_at_1000 + value: 0.099 + - type: mrr_at_1 + value: 15.4 + - type: mrr_at_3 + value: 21.099999999999998 + - type: mrr_at_5 + value: 23.380000000000003 + - type: mrr_at_10 + value: 25.5087 + - type: mrr_at_20 + value: 26.5332 + - type: mrr_at_100 + value: 27.0822 + - type: mrr_at_1000 + value: 27.1358 + - type: nauc_ndcg_at_1_max + value: 15.7645 + - type: nauc_ndcg_at_1_std + value: -8.4668 + - type: nauc_ndcg_at_1_diff1 + value: 38.0187 + - type: nauc_ndcg_at_3_max + value: 14.791799999999999 + - type: nauc_ndcg_at_3_std + value: -11.6736 + - type: nauc_ndcg_at_3_diff1 + value: 24.288899999999998 + - type: nauc_ndcg_at_5_max + value: 17.9426 + - type: nauc_ndcg_at_5_std + value: -11.1099 + - type: nauc_ndcg_at_5_diff1 + value: 18.8892 + - type: nauc_ndcg_at_10_max + value: 18.3537 + - type: nauc_ndcg_at_10_std + value: -9.0621 + - type: nauc_ndcg_at_10_diff1 + value: 17.6054 + - type: nauc_ndcg_at_20_max + value: 19.9156 + - type: nauc_ndcg_at_20_std + value: -6.926699999999999 + - type: nauc_ndcg_at_20_diff1 + value: 16.125 + - type: nauc_ndcg_at_100_max + value: 19.527900000000002 + - type: nauc_ndcg_at_100_std + value: -5.9748 + - type: nauc_ndcg_at_100_diff1 + value: 18.8697 + - type: nauc_ndcg_at_1000_max + value: 18.6624 + - type: nauc_ndcg_at_1000_std + value: -7.6636999999999995 + - type: nauc_ndcg_at_1000_diff1 + value: 20.2624 + - type: nauc_map_at_1_max + value: 15.7645 + - type: nauc_map_at_1_std + value: -8.4668 + - type: nauc_map_at_1_diff1 + value: 38.0187 + - type: nauc_map_at_3_max + value: 14.932200000000002 + - type: nauc_map_at_3_std + value: -11.2233 + - type: nauc_map_at_3_diff1 + value: 27.254800000000003 + - type: nauc_map_at_5_max + value: 16.700599999999998 + - type: nauc_map_at_5_std + value: -10.9701 + - type: nauc_map_at_5_diff1 + value: 23.9832 + - type: nauc_map_at_10_max + value: 16.947200000000002 + - type: nauc_map_at_10_std + value: -9.896099999999999 + - type: nauc_map_at_10_diff1 + value: 23.4428 + - type: nauc_map_at_20_max + value: 17.3857 + - type: nauc_map_at_20_std + value: -9.2728 + - type: nauc_map_at_20_diff1 + value: 23.1321 + - type: nauc_map_at_100_max + value: 17.3462 + - type: nauc_map_at_100_std + value: -9.2043 + - type: nauc_map_at_100_diff1 + value: 23.5583 + - type: nauc_map_at_1000_max + value: 17.3214 + - type: nauc_map_at_1000_std + value: -9.2627 + - type: nauc_map_at_1000_diff1 + value: 23.6455 + - type: nauc_recall_at_1_max + value: 15.7645 + - type: nauc_recall_at_1_std + value: -8.4668 + - type: nauc_recall_at_1_diff1 + value: 38.0187 + - type: nauc_recall_at_3_max + value: 14.4809 + - type: nauc_recall_at_3_std + value: -12.664700000000002 + - type: nauc_recall_at_3_diff1 + value: 17.275199999999998 + - type: nauc_recall_at_5_max + value: 21.2405 + - type: nauc_recall_at_5_std + value: -11.2278 + - type: nauc_recall_at_5_diff1 + value: 6.6622 + - type: nauc_recall_at_10_max + value: 22.3474 + - type: nauc_recall_at_10_std + value: -6.399299999999999 + - type: nauc_recall_at_10_diff1 + value: 2.0452000000000004 + - type: nauc_recall_at_20_max + value: 30.1398 + - type: nauc_recall_at_20_std + value: 3.3263000000000003 + - type: nauc_recall_at_20_diff1 + value: -9.3067 + - type: nauc_recall_at_100_max + value: 37.6654 + - type: nauc_recall_at_100_std + value: 30.699700000000004 + - type: nauc_recall_at_100_diff1 + value: -8.959999999999999 + - type: nauc_recall_at_1000_max + value: 47.3389 + - type: nauc_recall_at_1000_std + value: 95.6427 + - type: nauc_recall_at_1000_diff1 + value: -102.10079999999999 + - type: nauc_precision_at_1_max + value: 15.7645 + - type: nauc_precision_at_1_std + value: -8.4668 + - type: nauc_precision_at_1_diff1 + value: 38.0187 + - type: nauc_precision_at_3_max + value: 14.4809 + - type: nauc_precision_at_3_std + value: -12.664700000000002 + - type: nauc_precision_at_3_diff1 + value: 17.275199999999998 + - type: nauc_precision_at_5_max + value: 21.2405 + - type: nauc_precision_at_5_std + value: -11.2278 + - type: nauc_precision_at_5_diff1 + value: 6.6622 + - type: nauc_precision_at_10_max + value: 22.3474 + - type: nauc_precision_at_10_std + value: -6.399299999999999 + - type: nauc_precision_at_10_diff1 + value: 2.0452000000000004 + - type: nauc_precision_at_20_max + value: 30.1398 + - type: nauc_precision_at_20_std + value: 3.3263000000000003 + - type: nauc_precision_at_20_diff1 + value: -9.3067 + - type: nauc_precision_at_100_max + value: 37.6654 + - type: nauc_precision_at_100_std + value: 30.699700000000004 + - type: nauc_precision_at_100_diff1 + value: -8.959999999999999 + - type: nauc_precision_at_1000_max + value: 47.3389 + - type: nauc_precision_at_1000_std + value: 95.6427 + - type: nauc_precision_at_1000_diff1 + value: -102.10079999999999 + - type: nauc_mrr_at_1_max + value: 15.059800000000001 + - type: nauc_mrr_at_1_std + value: -17.3443 + - type: nauc_mrr_at_1_diff1 + value: 34.5918 + - type: nauc_mrr_at_3_max + value: 15.5076 + - type: nauc_mrr_at_3_std + value: -16.3353 + - type: nauc_mrr_at_3_diff1 + value: 27.414899999999996 + - type: nauc_mrr_at_5_max + value: 15.033299999999999 + - type: nauc_mrr_at_5_std + value: -16.0288 + - type: nauc_mrr_at_5_diff1 + value: 25.4198 + - type: nauc_mrr_at_10_max + value: 15.7434 + - type: nauc_mrr_at_10_std + value: -14.8923 + - type: nauc_mrr_at_10_diff1 + value: 23.6099 + - type: nauc_mrr_at_20_max + value: 16.2588 + - type: nauc_mrr_at_20_std + value: -14.5306 + - type: nauc_mrr_at_20_diff1 + value: 23.718700000000002 + - type: nauc_mrr_at_100_max + value: 16.2196 + - type: nauc_mrr_at_100_std + value: -14.4928 + - type: nauc_mrr_at_100_diff1 + value: 24.017 + - type: nauc_mrr_at_1000_max + value: 16.1885 + - type: nauc_mrr_at_1000_std + value: -14.5629 + - type: nauc_mrr_at_1000_diff1 + value: 24.0998 + - type: main_score + value: 33.452 + task: + type: Retrieval + - dataset: + config: default + name: MTEB DBPedia (default) + revision: c0f706b76e590d620bd6618b3ca8efdd34e2d659 + split: test + type: mteb/dbpedia + metrics: + - type: ndcg_at_1 + value: 48.75 + - type: ndcg_at_3 + value: 40.266000000000005 + - type: ndcg_at_5 + value: 37.034 + - type: ndcg_at_10 + value: 34.565 + - type: ndcg_at_20 + value: 34.013 + - type: ndcg_at_100 + value: 39.006 + - type: ndcg_at_1000 + value: 46.64 + - type: map_at_1 + value: 7.866 + - type: map_at_3 + value: 12.145999999999999 + - type: map_at_5 + value: 13.874 + - type: map_at_10 + value: 16.02 + - type: map_at_20 + value: 18.183 + - type: map_at_100 + value: 21.775 + - type: map_at_1000 + value: 23.203 + - type: recall_at_1 + value: 7.866 + - type: recall_at_3 + value: 13.700000000000001 + - type: recall_at_5 + value: 16.683 + - type: recall_at_10 + value: 21.059 + - type: recall_at_20 + value: 27.045 + - type: recall_at_100 + value: 45.236 + - type: recall_at_1000 + value: 69.867 + - type: precision_at_1 + value: 60.5 + - type: precision_at_3 + value: 44.083 + - type: precision_at_5 + value: 35.449999999999996 + - type: precision_at_10 + value: 26.400000000000002 + - type: precision_at_20 + value: 19.75 + - type: precision_at_100 + value: 8.472 + - type: precision_at_1000 + value: 1.822 + - type: mrr_at_1 + value: 60.5 + - type: mrr_at_3 + value: 67.625 + - type: mrr_at_5 + value: 68.4625 + - type: mrr_at_10 + value: 69.4092 + - type: mrr_at_20 + value: 69.6644 + - type: mrr_at_100 + value: 69.8187 + - type: mrr_at_1000 + value: 69.8284 + - type: nauc_ndcg_at_1_max + value: 27.385199999999998 + - type: nauc_ndcg_at_1_std + value: 15.502199999999998 + - type: nauc_ndcg_at_1_diff1 + value: 40.3474 + - type: nauc_ndcg_at_3_max + value: 23.691100000000002 + - type: nauc_ndcg_at_3_std + value: 17.8766 + - type: nauc_ndcg_at_3_diff1 + value: 26.1322 + - type: nauc_ndcg_at_5_max + value: 21.908 + - type: nauc_ndcg_at_5_std + value: 16.5012 + - type: nauc_ndcg_at_5_diff1 + value: 24.9377 + - type: nauc_ndcg_at_10_max + value: 21.5239 + - type: nauc_ndcg_at_10_std + value: 15.327399999999999 + - type: nauc_ndcg_at_10_diff1 + value: 25.0379 + - type: nauc_ndcg_at_20_max + value: 18.6445 + - type: nauc_ndcg_at_20_std + value: 10.4816 + - type: nauc_ndcg_at_20_diff1 + value: 24.5885 + - type: nauc_ndcg_at_100_max + value: 21.7258 + - type: nauc_ndcg_at_100_std + value: 14.514199999999999 + - type: nauc_ndcg_at_100_diff1 + value: 21.6285 + - type: nauc_ndcg_at_1000_max + value: 25.515 + - type: nauc_ndcg_at_1000_std + value: 23.278499999999998 + - type: nauc_ndcg_at_1000_diff1 + value: 21.3373 + - type: nauc_map_at_1_max + value: 2.911 + - type: nauc_map_at_1_std + value: -23.3734 + - type: nauc_map_at_1_diff1 + value: 31.251099999999997 + - type: nauc_map_at_3_max + value: 6.7765 + - type: nauc_map_at_3_std + value: -21.1466 + - type: nauc_map_at_3_diff1 + value: 26.6096 + - type: nauc_map_at_5_max + value: 7.2574 + - type: nauc_map_at_5_std + value: -18.0369 + - type: nauc_map_at_5_diff1 + value: 24.0648 + - type: nauc_map_at_10_max + value: 11.669699999999999 + - type: nauc_map_at_10_std + value: -10.5142 + - type: nauc_map_at_10_diff1 + value: 23.289099999999998 + - type: nauc_map_at_20_max + value: 13.9376 + - type: nauc_map_at_20_std + value: -4.1179 + - type: nauc_map_at_20_diff1 + value: 22.9493 + - type: nauc_map_at_100_max + value: 18.756600000000002 + - type: nauc_map_at_100_std + value: 7.5601 + - type: nauc_map_at_100_diff1 + value: 21.1962 + - type: nauc_map_at_1000_max + value: 20.4084 + - type: nauc_map_at_1000_std + value: 10.7807 + - type: nauc_map_at_1000_diff1 + value: 21.6074 + - type: nauc_recall_at_1_max + value: 2.911 + - type: nauc_recall_at_1_std + value: -23.3734 + - type: nauc_recall_at_1_diff1 + value: 31.251099999999997 + - type: nauc_recall_at_3_max + value: 5.9628 + - type: nauc_recall_at_3_std + value: -21.7657 + - type: nauc_recall_at_3_diff1 + value: 22.1779 + - type: nauc_recall_at_5_max + value: 4.2336 + - type: nauc_recall_at_5_std + value: -19.872 + - type: nauc_recall_at_5_diff1 + value: 17.4799 + - type: nauc_recall_at_10_max + value: 9.376900000000001 + - type: nauc_recall_at_10_std + value: -12.3596 + - type: nauc_recall_at_10_diff1 + value: 15.801100000000002 + - type: nauc_recall_at_20_max + value: 11.2098 + - type: nauc_recall_at_20_std + value: -6.471699999999999 + - type: nauc_recall_at_20_diff1 + value: 15.1155 + - type: nauc_recall_at_100_max + value: 16.7433 + - type: nauc_recall_at_100_std + value: 12.2849 + - type: nauc_recall_at_100_diff1 + value: 6.908499999999999 + - type: nauc_recall_at_1000_max + value: 18.6941 + - type: nauc_recall_at_1000_std + value: 25.2521 + - type: nauc_recall_at_1000_diff1 + value: 1.0488000000000002 + - type: nauc_precision_at_1_max + value: 39.5387 + - type: nauc_precision_at_1_std + value: 23.244600000000002 + - type: nauc_precision_at_1_diff1 + value: 50.275499999999994 + - type: nauc_precision_at_3_max + value: 32.3641 + - type: nauc_precision_at_3_std + value: 34.4136 + - type: nauc_precision_at_3_diff1 + value: 17.316200000000002 + - type: nauc_precision_at_5_max + value: 29.9613 + - type: nauc_precision_at_5_std + value: 39.3271 + - type: nauc_precision_at_5_diff1 + value: 13.352 + - type: nauc_precision_at_10_max + value: 29.5821 + - type: nauc_precision_at_10_std + value: 48.0976 + - type: nauc_precision_at_10_diff1 + value: 9.610000000000001 + - type: nauc_precision_at_20_max + value: 25.5555 + - type: nauc_precision_at_20_std + value: 49.3622 + - type: nauc_precision_at_20_diff1 + value: 8.0656 + - type: nauc_precision_at_100_max + value: 24.3874 + - type: nauc_precision_at_100_std + value: 49.613600000000005 + - type: nauc_precision_at_100_diff1 + value: 4.1512 + - type: nauc_precision_at_1000_max + value: 16.0014 + - type: nauc_precision_at_1000_std + value: 28.3243 + - type: nauc_precision_at_1000_diff1 + value: 11.5068 + - type: nauc_mrr_at_1_max + value: 39.5387 + - type: nauc_mrr_at_1_std + value: 23.244600000000002 + - type: nauc_mrr_at_1_diff1 + value: 50.275499999999994 + - type: nauc_mrr_at_3_max + value: 44.3328 + - type: nauc_mrr_at_3_std + value: 29.595900000000004 + - type: nauc_mrr_at_3_diff1 + value: 47.0929 + - type: nauc_mrr_at_5_max + value: 43.6678 + - type: nauc_mrr_at_5_std + value: 29.219299999999997 + - type: nauc_mrr_at_5_diff1 + value: 47.7731 + - type: nauc_mrr_at_10_max + value: 43.1409 + - type: nauc_mrr_at_10_std + value: 29.5283 + - type: nauc_mrr_at_10_diff1 + value: 47.7777 + - type: nauc_mrr_at_20_max + value: 43.2155 + - type: nauc_mrr_at_20_std + value: 29.378999999999998 + - type: nauc_mrr_at_20_diff1 + value: 47.826800000000006 + - type: nauc_mrr_at_100_max + value: 43.2448 + - type: nauc_mrr_at_100_std + value: 29.385 + - type: nauc_mrr_at_100_diff1 + value: 47.7931 + - type: nauc_mrr_at_1000_max + value: 43.2316 + - type: nauc_mrr_at_1000_std + value: 29.3645 + - type: nauc_mrr_at_1000_diff1 + value: 47.7958 + - type: main_score + value: 34.565 + task: + type: Retrieval + - dataset: + config: default + name: MTEB EmotionClassification (default) + revision: 4f58c6b202a23cf9a4da393831edf4f9183cad37 + split: test + type: mteb/emotion + metrics: + - type: accuracy + value: 36.449999999999996 + - type: f1 + value: 32.3042 + - type: f1_weighted + value: 38.7818 + - type: main_score + value: 36.449999999999996 + task: + type: Classification + - dataset: + config: default + name: MTEB FEVER (default) + revision: bea83ef9e8fb933d90a2f1d5515737465d613e12 + split: test + type: mteb/fever + metrics: + - type: ndcg_at_1 + value: 77.93299999999999 + - type: ndcg_at_3 + value: 83.146 + - type: ndcg_at_5 + value: 84.188 + - type: ndcg_at_10 + value: 84.932 + - type: ndcg_at_20 + value: 85.187 + - type: ndcg_at_100 + value: 85.452 + - type: ndcg_at_1000 + value: 85.68599999999999 + - type: map_at_1 + value: 72.173 + - type: map_at_3 + value: 79.618 + - type: map_at_5 + value: 80.32000000000001 + - type: map_at_10 + value: 80.674 + - type: map_at_20 + value: 80.762 + - type: map_at_100 + value: 80.81 + - type: map_at_1000 + value: 80.822 + - type: recall_at_1 + value: 72.173 + - type: recall_at_3 + value: 87.804 + - type: recall_at_5 + value: 90.556 + - type: recall_at_10 + value: 92.869 + - type: recall_at_20 + value: 93.768 + - type: recall_at_100 + value: 95.00699999999999 + - type: recall_at_1000 + value: 96.504 + - type: precision_at_1 + value: 77.93299999999999 + - type: precision_at_3 + value: 31.828 + - type: precision_at_5 + value: 19.727 + - type: precision_at_10 + value: 10.135 + - type: precision_at_20 + value: 5.136 + - type: precision_at_100 + value: 1.049 + - type: precision_at_1000 + value: 0.109 + - type: mrr_at_1 + value: 77.9328 + - type: mrr_at_3 + value: 85.221 + - type: mrr_at_5 + value: 85.8076 + - type: mrr_at_10 + value: 86.0963 + - type: mrr_at_20 + value: 86.1448 + - type: mrr_at_100 + value: 86.1622 + - type: mrr_at_1000 + value: 86.1631 + - type: nauc_ndcg_at_1_max + value: 27.804499999999997 + - type: nauc_ndcg_at_1_std + value: -31.1045 + - type: nauc_ndcg_at_1_diff1 + value: 66.6633 + - type: nauc_ndcg_at_3_max + value: 21.6576 + - type: nauc_ndcg_at_3_std + value: -24.3372 + - type: nauc_ndcg_at_3_diff1 + value: 48.9088 + - type: nauc_ndcg_at_5_max + value: 20.612 + - type: nauc_ndcg_at_5_std + value: -23.8007 + - type: nauc_ndcg_at_5_diff1 + value: 48.0635 + - type: nauc_ndcg_at_10_max + value: 19.6463 + - type: nauc_ndcg_at_10_std + value: -22.5941 + - type: nauc_ndcg_at_10_diff1 + value: 47.5561 + - type: nauc_ndcg_at_20_max + value: 19.5443 + - type: nauc_ndcg_at_20_std + value: -21.998 + - type: nauc_ndcg_at_20_diff1 + value: 47.664699999999996 + - type: nauc_ndcg_at_100_max + value: 19.2285 + - type: nauc_ndcg_at_100_std + value: -21.6826 + - type: nauc_ndcg_at_100_diff1 + value: 47.897099999999995 + - type: nauc_ndcg_at_1000_max + value: 19.5578 + - type: nauc_ndcg_at_1000_std + value: -21.9412 + - type: nauc_ndcg_at_1000_diff1 + value: 48.361 + - type: nauc_map_at_1_max + value: 20.3735 + - type: nauc_map_at_1_std + value: -24.7274 + - type: nauc_map_at_1_diff1 + value: 54.148399999999995 + - type: nauc_map_at_3_max + value: 19.3166 + - type: nauc_map_at_3_std + value: -23.171 + - type: nauc_map_at_3_diff1 + value: 48.254000000000005 + - type: nauc_map_at_5_max + value: 19.158900000000003 + - type: nauc_map_at_5_std + value: -22.966900000000003 + - type: nauc_map_at_5_diff1 + value: 48.0877 + - type: nauc_map_at_10_max + value: 18.8745 + - type: nauc_map_at_10_std + value: -22.5913 + - type: nauc_map_at_10_diff1 + value: 47.957899999999995 + - type: nauc_map_at_20_max + value: 18.895200000000003 + - type: nauc_map_at_20_std + value: -22.4542 + - type: nauc_map_at_20_diff1 + value: 48.0047 + - type: nauc_map_at_100_max + value: 18.8722 + - type: nauc_map_at_100_std + value: -22.3984 + - type: nauc_map_at_100_diff1 + value: 48.0394 + - type: nauc_map_at_1000_max + value: 18.8824 + - type: nauc_map_at_1000_std + value: -22.4034 + - type: nauc_map_at_1000_diff1 + value: 48.0533 + - type: nauc_recall_at_1_max + value: 20.3735 + - type: nauc_recall_at_1_std + value: -24.7274 + - type: nauc_recall_at_1_diff1 + value: 54.148399999999995 + - type: nauc_recall_at_3_max + value: 15.2387 + - type: nauc_recall_at_3_std + value: -17.3947 + - type: nauc_recall_at_3_diff1 + value: 30.6589 + - type: nauc_recall_at_5_max + value: 11.4037 + - type: nauc_recall_at_5_std + value: -14.3603 + - type: nauc_recall_at_5_diff1 + value: 23.7356 + - type: nauc_recall_at_10_max + value: 3.8233 + - type: nauc_recall_at_10_std + value: -4.6399 + - type: nauc_recall_at_10_diff1 + value: 13.8514 + - type: nauc_recall_at_20_max + value: 0.3939 + - type: nauc_recall_at_20_std + value: 2.4212000000000002 + - type: nauc_recall_at_20_diff1 + value: 10.110800000000001 + - type: nauc_recall_at_100_max + value: -8.9768 + - type: nauc_recall_at_100_std + value: 11.2598 + - type: nauc_recall_at_100_diff1 + value: 4.6753 + - type: nauc_recall_at_1000_max + value: -13.494800000000001 + - type: nauc_recall_at_1000_std + value: 17.2306 + - type: nauc_recall_at_1000_diff1 + value: 0.0856 + - type: nauc_precision_at_1_max + value: 27.804499999999997 + - type: nauc_precision_at_1_std + value: -31.1045 + - type: nauc_precision_at_1_diff1 + value: 66.6633 + - type: nauc_precision_at_3_max + value: 25.660899999999998 + - type: nauc_precision_at_3_std + value: -22.0243 + - type: nauc_precision_at_3_diff1 + value: 34.5966 + - type: nauc_precision_at_5_max + value: 22.4777 + - type: nauc_precision_at_5_std + value: -14.9469 + - type: nauc_precision_at_5_diff1 + value: 20.9233 + - type: nauc_precision_at_10_max + value: 13.7882 + - type: nauc_precision_at_10_std + value: -0.1941 + - type: nauc_precision_at_10_diff1 + value: 2.5737 + - type: nauc_precision_at_20_max + value: 10.422099999999999 + - type: nauc_precision_at_20_std + value: 8.518 + - type: nauc_precision_at_20_diff1 + value: -4.2715000000000005 + - type: nauc_precision_at_100_max + value: 3.8884000000000003 + - type: nauc_precision_at_100_std + value: 14.529800000000002 + - type: nauc_precision_at_100_diff1 + value: -10.066 + - type: nauc_precision_at_1000_max + value: 5.5056 + - type: nauc_precision_at_1000_std + value: 10.3948 + - type: nauc_precision_at_1000_diff1 + value: -9.5234 + - type: nauc_mrr_at_1_max + value: 27.804499999999997 + - type: nauc_mrr_at_1_std + value: -31.1045 + - type: nauc_mrr_at_1_diff1 + value: 66.6633 + - type: nauc_mrr_at_3_max + value: 30.593500000000002 + - type: nauc_mrr_at_3_std + value: -31.844499999999996 + - type: nauc_mrr_at_3_diff1 + value: 63.571 + - type: nauc_mrr_at_5_max + value: 30.544700000000002 + - type: nauc_mrr_at_5_std + value: -32.0369 + - type: nauc_mrr_at_5_diff1 + value: 63.8464 + - type: nauc_mrr_at_10_max + value: 30.459000000000003 + - type: nauc_mrr_at_10_std + value: -31.799500000000002 + - type: nauc_mrr_at_10_diff1 + value: 64.0984 + - type: nauc_mrr_at_20_max + value: 30.3871 + - type: nauc_mrr_at_20_std + value: -31.6429 + - type: nauc_mrr_at_20_diff1 + value: 64.1444 + - type: nauc_mrr_at_100_max + value: 30.324099999999998 + - type: nauc_mrr_at_100_std + value: -31.629800000000003 + - type: nauc_mrr_at_100_diff1 + value: 64.163 + - type: nauc_mrr_at_1000_max + value: 30.3201 + - type: nauc_mrr_at_1000_std + value: -31.6352 + - type: nauc_mrr_at_1000_diff1 + value: 64.1637 + - type: main_score + value: 84.932 + task: + type: Retrieval + - dataset: + config: default + name: MTEB FiQA2018 (default) + revision: 27a168819829fe9bcd655c2df245fb19452e8e06 + split: test + type: mteb/fiqa + metrics: + - type: ndcg_at_1 + value: 34.259 + - type: ndcg_at_3 + value: 32.14 + - type: ndcg_at_5 + value: 33.391 + - type: ndcg_at_10 + value: 35.663 + - type: ndcg_at_20 + value: 38.193 + - type: ndcg_at_100 + value: 42.232 + - type: ndcg_at_1000 + value: 45.595 + - type: map_at_1 + value: 17.124 + - type: map_at_3 + value: 24.359 + - type: map_at_5 + value: 26.532 + - type: map_at_10 + value: 28.183000000000003 + - type: map_at_20 + value: 29.119 + - type: map_at_100 + value: 29.881 + - type: map_at_1000 + value: 30.070000000000004 + - type: recall_at_1 + value: 17.124 + - type: recall_at_3 + value: 29.488999999999997 + - type: recall_at_5 + value: 35.436 + - type: recall_at_10 + value: 42.665 + - type: recall_at_20 + value: 50.381 + - type: recall_at_100 + value: 67.364 + - type: recall_at_1000 + value: 87.315 + - type: precision_at_1 + value: 34.259 + - type: precision_at_3 + value: 21.399 + - type: precision_at_5 + value: 15.926000000000002 + - type: precision_at_10 + value: 9.907 + - type: precision_at_20 + value: 6.026 + - type: precision_at_100 + value: 1.637 + - type: precision_at_1000 + value: 0.22599999999999998 + - type: mrr_at_1 + value: 34.259299999999996 + - type: mrr_at_3 + value: 40.7922 + - type: mrr_at_5 + value: 42.1811 + - type: mrr_at_10 + value: 43.1663 + - type: mrr_at_20 + value: 43.684400000000004 + - type: mrr_at_100 + value: 44.079 + - type: mrr_at_1000 + value: 44.1277 + - type: nauc_ndcg_at_1_max + value: 45.5993 + - type: nauc_ndcg_at_1_std + value: 4.2730999999999995 + - type: nauc_ndcg_at_1_diff1 + value: 51.0941 + - type: nauc_ndcg_at_3_max + value: 38.6082 + - type: nauc_ndcg_at_3_std + value: 1.7973 + - type: nauc_ndcg_at_3_diff1 + value: 41.556599999999996 + - type: nauc_ndcg_at_5_max + value: 37.0326 + - type: nauc_ndcg_at_5_std + value: 3.5555000000000003 + - type: nauc_ndcg_at_5_diff1 + value: 41.166599999999995 + - type: nauc_ndcg_at_10_max + value: 36.8257 + - type: nauc_ndcg_at_10_std + value: 4.6765 + - type: nauc_ndcg_at_10_diff1 + value: 40.7039 + - type: nauc_ndcg_at_20_max + value: 37.9542 + - type: nauc_ndcg_at_20_std + value: 6.2273000000000005 + - type: nauc_ndcg_at_20_diff1 + value: 40.7126 + - type: nauc_ndcg_at_100_max + value: 40.029399999999995 + - type: nauc_ndcg_at_100_std + value: 8.8925 + - type: nauc_ndcg_at_100_diff1 + value: 40.8749 + - type: nauc_ndcg_at_1000_max + value: 41.0995 + - type: nauc_ndcg_at_1000_std + value: 9.055399999999999 + - type: nauc_ndcg_at_1000_diff1 + value: 42.0999 + - type: nauc_map_at_1_max + value: 29.1034 + - type: nauc_map_at_1_std + value: -1.3329 + - type: nauc_map_at_1_diff1 + value: 49.6713 + - type: nauc_map_at_3_max + value: 31.2555 + - type: nauc_map_at_3_std + value: -1.2727 + - type: nauc_map_at_3_diff1 + value: 42.8671 + - type: nauc_map_at_5_max + value: 32.7495 + - type: nauc_map_at_5_std + value: 0.4463 + - type: nauc_map_at_5_diff1 + value: 42.3138 + - type: nauc_map_at_10_max + value: 34.0564 + - type: nauc_map_at_10_std + value: 1.8785 + - type: nauc_map_at_10_diff1 + value: 41.9711 + - type: nauc_map_at_20_max + value: 34.7449 + - type: nauc_map_at_20_std + value: 2.6273 + - type: nauc_map_at_20_diff1 + value: 41.9563 + - type: nauc_map_at_100_max + value: 35.3724 + - type: nauc_map_at_100_std + value: 3.1910000000000003 + - type: nauc_map_at_100_diff1 + value: 41.990899999999996 + - type: nauc_map_at_1000_max + value: 35.4782 + - type: nauc_map_at_1000_std + value: 3.2302999999999997 + - type: nauc_map_at_1000_diff1 + value: 42.0484 + - type: nauc_recall_at_1_max + value: 29.1034 + - type: nauc_recall_at_1_std + value: -1.3329 + - type: nauc_recall_at_1_diff1 + value: 49.6713 + - type: nauc_recall_at_3_max + value: 28.3729 + - type: nauc_recall_at_3_std + value: 0.0225 + - type: nauc_recall_at_3_diff1 + value: 35.2655 + - type: nauc_recall_at_5_max + value: 28.0157 + - type: nauc_recall_at_5_std + value: 3.5967 + - type: nauc_recall_at_5_diff1 + value: 31.5507 + - type: nauc_recall_at_10_max + value: 28.0271 + - type: nauc_recall_at_10_std + value: 6.7875000000000005 + - type: nauc_recall_at_10_diff1 + value: 28.3267 + - type: nauc_recall_at_20_max + value: 30.2764 + - type: nauc_recall_at_20_std + value: 11.2697 + - type: nauc_recall_at_20_diff1 + value: 27.5277 + - type: nauc_recall_at_100_max + value: 33.2215 + - type: nauc_recall_at_100_std + value: 23.6362 + - type: nauc_recall_at_100_diff1 + value: 23.1851 + - type: nauc_recall_at_1000_max + value: 41.8199 + - type: nauc_recall_at_1000_std + value: 42.2866 + - type: nauc_recall_at_1000_diff1 + value: 29.341099999999997 + - type: nauc_precision_at_1_max + value: 45.5993 + - type: nauc_precision_at_1_std + value: 4.2730999999999995 + - type: nauc_precision_at_1_diff1 + value: 51.0941 + - type: nauc_precision_at_3_max + value: 40.541 + - type: nauc_precision_at_3_std + value: 3.6046 + - type: nauc_precision_at_3_diff1 + value: 29.2879 + - type: nauc_precision_at_5_max + value: 40.4116 + - type: nauc_precision_at_5_std + value: 9.523 + - type: nauc_precision_at_5_diff1 + value: 24.9572 + - type: nauc_precision_at_10_max + value: 39.7377 + - type: nauc_precision_at_10_std + value: 11.8076 + - type: nauc_precision_at_10_diff1 + value: 21.1979 + - type: nauc_precision_at_20_max + value: 40.1851 + - type: nauc_precision_at_20_std + value: 14.967 + - type: nauc_precision_at_20_diff1 + value: 19.0881 + - type: nauc_precision_at_100_max + value: 39.4474 + - type: nauc_precision_at_100_std + value: 19.6785 + - type: nauc_precision_at_100_diff1 + value: 12.6951 + - type: nauc_precision_at_1000_max + value: 32.071600000000004 + - type: nauc_precision_at_1000_std + value: 14.7899 + - type: nauc_precision_at_1000_diff1 + value: 7.456599999999999 + - type: nauc_mrr_at_1_max + value: 45.5993 + - type: nauc_mrr_at_1_std + value: 4.2730999999999995 + - type: nauc_mrr_at_1_diff1 + value: 51.0941 + - type: nauc_mrr_at_3_max + value: 45.5586 + - type: nauc_mrr_at_3_std + value: 5.6932 + - type: nauc_mrr_at_3_diff1 + value: 47.1359 + - type: nauc_mrr_at_5_max + value: 45.0408 + - type: nauc_mrr_at_5_std + value: 6.4838000000000005 + - type: nauc_mrr_at_5_diff1 + value: 46.4912 + - type: nauc_mrr_at_10_max + value: 44.9499 + - type: nauc_mrr_at_10_std + value: 6.6139 + - type: nauc_mrr_at_10_diff1 + value: 46.332699999999996 + - type: nauc_mrr_at_20_max + value: 45.063900000000004 + - type: nauc_mrr_at_20_std + value: 6.6114999999999995 + - type: nauc_mrr_at_20_diff1 + value: 46.3181 + - type: nauc_mrr_at_100_max + value: 45.2249 + - type: nauc_mrr_at_100_std + value: 6.8897 + - type: nauc_mrr_at_100_diff1 + value: 46.373799999999996 + - type: nauc_mrr_at_1000_max + value: 45.2235 + - type: nauc_mrr_at_1000_std + value: 6.8732 + - type: nauc_mrr_at_1000_diff1 + value: 46.399699999999996 + - type: main_score + value: 35.663 + task: + type: Retrieval + - dataset: + config: default + name: MTEB HotpotQA (default) + revision: ab518f4d6fcca38d87c25209f94beba119d02014 + split: test + type: mteb/hotpotqa + metrics: + - type: ndcg_at_1 + value: 75.908 + - type: ndcg_at_3 + value: 57.643 + - type: ndcg_at_5 + value: 59.689 + - type: ndcg_at_10 + value: 61.513 + - type: ndcg_at_20 + value: 62.721000000000004 + - type: ndcg_at_100 + value: 64.57000000000001 + - type: ndcg_at_1000 + value: 65.981 + - type: map_at_1 + value: 37.954 + - type: map_at_3 + value: 49.424 + - type: map_at_5 + value: 50.99399999999999 + - type: map_at_10 + value: 52.066 + - type: map_at_20 + value: 52.54600000000001 + - type: map_at_100 + value: 52.910000000000004 + - type: map_at_1000 + value: 52.981 + - type: recall_at_1 + value: 37.954 + - type: recall_at_3 + value: 53.201 + - type: recall_at_5 + value: 57.232000000000006 + - type: recall_at_10 + value: 61.82299999999999 + - type: recall_at_20 + value: 65.692 + - type: recall_at_100 + value: 73.896 + - type: recall_at_1000 + value: 83.255 + - type: precision_at_1 + value: 75.908 + - type: precision_at_3 + value: 35.467 + - type: precision_at_5 + value: 22.893 + - type: precision_at_10 + value: 12.365 + - type: precision_at_20 + value: 6.569 + - type: precision_at_100 + value: 1.478 + - type: precision_at_1000 + value: 0.167 + - type: mrr_at_1 + value: 75.90820000000001 + - type: mrr_at_3 + value: 80.5717 + - type: mrr_at_5 + value: 81.15299999999999 + - type: mrr_at_10 + value: 81.4709 + - type: mrr_at_20 + value: 81.6082 + - type: mrr_at_100 + value: 81.69239999999999 + - type: mrr_at_1000 + value: 81.7034 + - type: nauc_ndcg_at_1_max + value: 53.456199999999995 + - type: nauc_ndcg_at_1_std + value: -7.1338 + - type: nauc_ndcg_at_1_diff1 + value: 72.2296 + - type: nauc_ndcg_at_3_max + value: 30.760199999999998 + - type: nauc_ndcg_at_3_std + value: -3.1088999999999998 + - type: nauc_ndcg_at_3_diff1 + value: 29.957099999999997 + - type: nauc_ndcg_at_5_max + value: 29.404000000000003 + - type: nauc_ndcg_at_5_std + value: -1.8713 + - type: nauc_ndcg_at_5_diff1 + value: 27.3461 + - type: nauc_ndcg_at_10_max + value: 28.0841 + - type: nauc_ndcg_at_10_std + value: -0.8572 + - type: nauc_ndcg_at_10_diff1 + value: 25.1934 + - type: nauc_ndcg_at_20_max + value: 27.581099999999996 + - type: nauc_ndcg_at_20_std + value: -0.1989 + - type: nauc_ndcg_at_20_diff1 + value: 24.3724 + - type: nauc_ndcg_at_100_max + value: 27.0287 + - type: nauc_ndcg_at_100_std + value: 0.7972 + - type: nauc_ndcg_at_100_diff1 + value: 23.6936 + - type: nauc_ndcg_at_1000_max + value: 27.070800000000002 + - type: nauc_ndcg_at_1000_std + value: 0.8108000000000001 + - type: nauc_ndcg_at_1000_diff1 + value: 24.0546 + - type: nauc_map_at_1_max + value: 53.456199999999995 + - type: nauc_map_at_1_std + value: -7.1338 + - type: nauc_map_at_1_diff1 + value: 72.2296 + - type: nauc_map_at_3_max + value: 26.085199999999997 + - type: nauc_map_at_3_std + value: -3.3792999999999997 + - type: nauc_map_at_3_diff1 + value: 23.335900000000002 + - type: nauc_map_at_5_max + value: 25.2911 + - type: nauc_map_at_5_std + value: -2.6356 + - type: nauc_map_at_5_diff1 + value: 21.7569 + - type: nauc_map_at_10_max + value: 24.5926 + - type: nauc_map_at_10_std + value: -2.1178 + - type: nauc_map_at_10_diff1 + value: 20.6735 + - type: nauc_map_at_20_max + value: 24.479400000000002 + - type: nauc_map_at_20_std + value: -1.8454000000000002 + - type: nauc_map_at_20_diff1 + value: 20.4617 + - type: nauc_map_at_100_max + value: 24.390600000000003 + - type: nauc_map_at_100_std + value: -1.6625999999999999 + - type: nauc_map_at_100_diff1 + value: 20.3774 + - type: nauc_map_at_1000_max + value: 24.387900000000002 + - type: nauc_map_at_1000_std + value: -1.6534 + - type: nauc_map_at_1000_diff1 + value: 20.3887 + - type: nauc_recall_at_1_max + value: 53.456199999999995 + - type: nauc_recall_at_1_std + value: -7.1338 + - type: nauc_recall_at_1_diff1 + value: 72.2296 + - type: nauc_recall_at_3_max + value: 22.2324 + - type: nauc_recall_at_3_std + value: -1.4433 + - type: nauc_recall_at_3_diff1 + value: 14.944799999999999 + - type: nauc_recall_at_5_max + value: 19.1126 + - type: nauc_recall_at_5_std + value: 0.9252 + - type: nauc_recall_at_5_diff1 + value: 9.6723 + - type: nauc_recall_at_10_max + value: 15.4048 + - type: nauc_recall_at_10_std + value: 3.3196000000000003 + - type: nauc_recall_at_10_diff1 + value: 4.2059 + - type: nauc_recall_at_20_max + value: 12.7643 + - type: nauc_recall_at_20_std + value: 5.431699999999999 + - type: nauc_recall_at_20_diff1 + value: 0.46880000000000005 + - type: nauc_recall_at_100_max + value: 7.538 + - type: nauc_recall_at_100_std + value: 10.5696 + - type: nauc_recall_at_100_diff1 + value: -6.472300000000001 + - type: nauc_recall_at_1000_max + value: 1.7873 + - type: nauc_recall_at_1000_std + value: 13.6112 + - type: nauc_recall_at_1000_diff1 + value: -13.081000000000001 + - type: nauc_precision_at_1_max + value: 53.456199999999995 + - type: nauc_precision_at_1_std + value: -7.1338 + - type: nauc_precision_at_1_diff1 + value: 72.2296 + - type: nauc_precision_at_3_max + value: 22.2324 + - type: nauc_precision_at_3_std + value: -1.4433 + - type: nauc_precision_at_3_diff1 + value: 14.944799999999999 + - type: nauc_precision_at_5_max + value: 19.1126 + - type: nauc_precision_at_5_std + value: 0.9252 + - type: nauc_precision_at_5_diff1 + value: 9.6723 + - type: nauc_precision_at_10_max + value: 15.4048 + - type: nauc_precision_at_10_std + value: 3.3196000000000003 + - type: nauc_precision_at_10_diff1 + value: 4.2059 + - type: nauc_precision_at_20_max + value: 12.7643 + - type: nauc_precision_at_20_std + value: 5.431699999999999 + - type: nauc_precision_at_20_diff1 + value: 0.46880000000000005 + - type: nauc_precision_at_100_max + value: 7.538 + - type: nauc_precision_at_100_std + value: 10.5696 + - type: nauc_precision_at_100_diff1 + value: -6.472300000000001 + - type: nauc_precision_at_1000_max + value: 1.7873 + - type: nauc_precision_at_1000_std + value: 13.6112 + - type: nauc_precision_at_1000_diff1 + value: -13.081000000000001 + - type: nauc_mrr_at_1_max + value: 53.456199999999995 + - type: nauc_mrr_at_1_std + value: -7.1338 + - type: nauc_mrr_at_1_diff1 + value: 72.2296 + - type: nauc_mrr_at_3_max + value: 54.94369999999999 + - type: nauc_mrr_at_3_std + value: -5.0057 + - type: nauc_mrr_at_3_diff1 + value: 69.6774 + - type: nauc_mrr_at_5_max + value: 54.970699999999994 + - type: nauc_mrr_at_5_std + value: -4.3104000000000005 + - type: nauc_mrr_at_5_diff1 + value: 69.4618 + - type: nauc_mrr_at_10_max + value: 55.01970000000001 + - type: nauc_mrr_at_10_std + value: -4.0596 + - type: nauc_mrr_at_10_diff1 + value: 69.435 + - type: nauc_mrr_at_20_max + value: 54.9824 + - type: nauc_mrr_at_20_std + value: -4.1227 + - type: nauc_mrr_at_20_diff1 + value: 69.4712 + - type: nauc_mrr_at_100_max + value: 54.9588 + - type: nauc_mrr_at_100_std + value: -4.1325 + - type: nauc_mrr_at_100_diff1 + value: 69.498 + - type: nauc_mrr_at_1000_max + value: 54.95179999999999 + - type: nauc_mrr_at_1000_std + value: -4.1442 + - type: nauc_mrr_at_1000_diff1 + value: 69.503 + - type: main_score + value: 61.513 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ImdbClassification (default) + revision: 3d86128a09e091d6018b6d26cad27f2739fc2db7 + split: test + type: mteb/imdb + metrics: + - type: accuracy + value: 63.0232 + - type: f1 + value: 62.8137 + - type: f1_weighted + value: 62.8137 + - type: ap + value: 58.377199999999995 + - type: ap_weighted + value: 58.377199999999995 + - type: main_score + value: 63.0232 + task: + type: Classification + - dataset: + config: ar + name: MTEB MIRACLRetrieval (ar) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 57.459 + - type: ndcg_at_3 + value: 58.162000000000006 + - type: ndcg_at_5 + value: 60.831 + - type: ndcg_at_10 + value: 64.238 + - type: ndcg_at_20 + value: 66.455 + - type: ndcg_at_100 + value: 68.67 + - type: ndcg_at_1000 + value: 69.51 + - type: map_at_1 + value: 38.064 + - type: map_at_3 + value: 51.217999999999996 + - type: map_at_5 + value: 54.364999999999995 + - type: map_at_10 + value: 56.589999999999996 + - type: map_at_20 + value: 57.545 + - type: map_at_100 + value: 58.06400000000001 + - type: map_at_1000 + value: 58.111999999999995 + - type: recall_at_1 + value: 38.064 + - type: recall_at_3 + value: 58.618 + - type: recall_at_5 + value: 66.353 + - type: recall_at_10 + value: 75.098 + - type: recall_at_20 + value: 81.978 + - type: recall_at_100 + value: 91.203 + - type: recall_at_1000 + value: 96.706 + - type: precision_at_1 + value: 57.459 + - type: precision_at_3 + value: 32.965 + - type: precision_at_5 + value: 23.405 + - type: precision_at_10 + value: 13.816 + - type: precision_at_20 + value: 7.742 + - type: precision_at_100 + value: 1.7739999999999998 + - type: precision_at_1000 + value: 0.189 + - type: mrr_at_1 + value: 57.458600000000004 + - type: mrr_at_3 + value: 65.4523 + - type: mrr_at_5 + value: 66.6506 + - type: mrr_at_10 + value: 67.48100000000001 + - type: mrr_at_20 + value: 67.7522 + - type: mrr_at_100 + value: 67.88419999999999 + - type: mrr_at_1000 + value: 67.8972 + - type: nauc_ndcg_at_1_max + value: 38.2614 + - type: nauc_ndcg_at_1_std + value: 1.0798999999999999 + - type: nauc_ndcg_at_1_diff1 + value: 44.3159 + - type: nauc_ndcg_at_3_max + value: 35.7658 + - type: nauc_ndcg_at_3_std + value: -3.9097 + - type: nauc_ndcg_at_3_diff1 + value: 36.8009 + - type: nauc_ndcg_at_5_max + value: 37.7543 + - type: nauc_ndcg_at_5_std + value: -2.7727999999999997 + - type: nauc_ndcg_at_5_diff1 + value: 36.8992 + - type: nauc_ndcg_at_10_max + value: 39.9339 + - type: nauc_ndcg_at_10_std + value: -0.2843 + - type: nauc_ndcg_at_10_diff1 + value: 36.7359 + - type: nauc_ndcg_at_20_max + value: 40.9231 + - type: nauc_ndcg_at_20_std + value: 1.5467 + - type: nauc_ndcg_at_20_diff1 + value: 36.5693 + - type: nauc_ndcg_at_100_max + value: 41.554 + - type: nauc_ndcg_at_100_std + value: 3.7470999999999997 + - type: nauc_ndcg_at_100_diff1 + value: 36.6323 + - type: nauc_ndcg_at_1000_max + value: 41.1969 + - type: nauc_ndcg_at_1000_std + value: 2.9972 + - type: nauc_ndcg_at_1000_diff1 + value: 37.1419 + - type: nauc_map_at_1_max + value: 21.1612 + - type: nauc_map_at_1_std + value: -11.2901 + - type: nauc_map_at_1_diff1 + value: 43.8572 + - type: nauc_map_at_3_max + value: 31.0197 + - type: nauc_map_at_3_std + value: -7.5985 + - type: nauc_map_at_3_diff1 + value: 38.0396 + - type: nauc_map_at_5_max + value: 33.8261 + - type: nauc_map_at_5_std + value: -5.501 + - type: nauc_map_at_5_diff1 + value: 37.2243 + - type: nauc_map_at_10_max + value: 35.5222 + - type: nauc_map_at_10_std + value: -3.7351 + - type: nauc_map_at_10_diff1 + value: 36.8849 + - type: nauc_map_at_20_max + value: 36.0478 + - type: nauc_map_at_20_std + value: -2.9566 + - type: nauc_map_at_20_diff1 + value: 36.7755 + - type: nauc_map_at_100_max + value: 36.256 + - type: nauc_map_at_100_std + value: -2.455 + - type: nauc_map_at_100_diff1 + value: 36.778800000000004 + - type: nauc_map_at_1000_max + value: 36.249900000000004 + - type: nauc_map_at_1000_std + value: -2.4678999999999998 + - type: nauc_map_at_1000_diff1 + value: 36.7962 + - type: nauc_recall_at_1_max + value: 21.1612 + - type: nauc_recall_at_1_std + value: -11.2901 + - type: nauc_recall_at_1_diff1 + value: 43.8572 + - type: nauc_recall_at_3_max + value: 30.1126 + - type: nauc_recall_at_3_std + value: -8.705499999999999 + - type: nauc_recall_at_3_diff1 + value: 33.0274 + - type: nauc_recall_at_5_max + value: 35.5301 + - type: nauc_recall_at_5_std + value: -4.1692 + - type: nauc_recall_at_5_diff1 + value: 30.693900000000003 + - type: nauc_recall_at_10_max + value: 41.431200000000004 + - type: nauc_recall_at_10_std + value: 3.1441999999999997 + - type: nauc_recall_at_10_diff1 + value: 28.5864 + - type: nauc_recall_at_20_max + value: 46.097100000000005 + - type: nauc_recall_at_20_std + value: 10.93 + - type: nauc_recall_at_20_diff1 + value: 26.930100000000003 + - type: nauc_recall_at_100_max + value: 58.3395 + - type: nauc_recall_at_100_std + value: 40.328599999999994 + - type: nauc_recall_at_100_diff1 + value: 21.9273 + - type: nauc_recall_at_1000_max + value: 72.4689 + - type: nauc_recall_at_1000_std + value: 59.1972 + - type: nauc_recall_at_1000_diff1 + value: 27.697899999999997 + - type: nauc_precision_at_1_max + value: 38.2614 + - type: nauc_precision_at_1_std + value: 1.0798999999999999 + - type: nauc_precision_at_1_diff1 + value: 44.3159 + - type: nauc_precision_at_3_max + value: 35.755700000000004 + - type: nauc_precision_at_3_std + value: 11.9015 + - type: nauc_precision_at_3_diff1 + value: 8.3107 + - type: nauc_precision_at_5_max + value: 33.9849 + - type: nauc_precision_at_5_std + value: 16.7448 + - type: nauc_precision_at_5_diff1 + value: 0.6217999999999999 + - type: nauc_precision_at_10_max + value: 29.9323 + - type: nauc_precision_at_10_std + value: 21.601100000000002 + - type: nauc_precision_at_10_diff1 + value: -5.758900000000001 + - type: nauc_precision_at_20_max + value: 26.142100000000003 + - type: nauc_precision_at_20_std + value: 25.1079 + - type: nauc_precision_at_20_diff1 + value: -9.9798 + - type: nauc_precision_at_100_max + value: 19.456100000000003 + - type: nauc_precision_at_100_std + value: 28.674899999999997 + - type: nauc_precision_at_100_diff1 + value: -14.6005 + - type: nauc_precision_at_1000_max + value: 14.49 + - type: nauc_precision_at_1000_std + value: 25.480399999999996 + - type: nauc_precision_at_1000_diff1 + value: -15.570899999999998 + - type: nauc_mrr_at_1_max + value: 38.2614 + - type: nauc_mrr_at_1_std + value: 1.0798999999999999 + - type: nauc_mrr_at_1_diff1 + value: 44.3159 + - type: nauc_mrr_at_3_max + value: 42.2344 + - type: nauc_mrr_at_3_std + value: 1.9994 + - type: nauc_mrr_at_3_diff1 + value: 41.5794 + - type: nauc_mrr_at_5_max + value: 42.9754 + - type: nauc_mrr_at_5_std + value: 2.8443 + - type: nauc_mrr_at_5_diff1 + value: 41.5702 + - type: nauc_mrr_at_10_max + value: 43.0856 + - type: nauc_mrr_at_10_std + value: 3.1882 + - type: nauc_mrr_at_10_diff1 + value: 41.6792 + - type: nauc_mrr_at_20_max + value: 42.972300000000004 + - type: nauc_mrr_at_20_std + value: 3.2651 + - type: nauc_mrr_at_20_diff1 + value: 41.6405 + - type: nauc_mrr_at_100_max + value: 42.945499999999996 + - type: nauc_mrr_at_100_std + value: 3.3168 + - type: nauc_mrr_at_100_diff1 + value: 41.6818 + - type: nauc_mrr_at_1000_max + value: 42.9332 + - type: nauc_mrr_at_1000_std + value: 3.3009999999999997 + - type: nauc_mrr_at_1000_diff1 + value: 41.6879 + - type: main_score + value: 64.238 + task: + type: Retrieval + - dataset: + config: bn + name: MTEB MIRACLRetrieval (bn) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 60.341 + - type: ndcg_at_3 + value: 60.805 + - type: ndcg_at_5 + value: 64.486 + - type: ndcg_at_10 + value: 68.05499999999999 + - type: ndcg_at_20 + value: 69.914 + - type: ndcg_at_100 + value: 72.00800000000001 + - type: ndcg_at_1000 + value: 72.71600000000001 + - type: map_at_1 + value: 37.948 + - type: map_at_3 + value: 52.89 + - type: map_at_5 + value: 56.845 + - type: map_at_10 + value: 59.329 + - type: map_at_20 + value: 60.158 + - type: map_at_100 + value: 60.73 + - type: map_at_1000 + value: 60.778 + - type: recall_at_1 + value: 37.948 + - type: recall_at_3 + value: 61.095 + - type: recall_at_5 + value: 71.316 + - type: recall_at_10 + value: 80.609 + - type: recall_at_20 + value: 86.141 + - type: recall_at_100 + value: 94.305 + - type: recall_at_1000 + value: 98.625 + - type: precision_at_1 + value: 60.341 + - type: precision_at_3 + value: 36.172 + - type: precision_at_5 + value: 26.277 + - type: precision_at_10 + value: 15.595999999999998 + - type: precision_at_20 + value: 8.552 + - type: precision_at_100 + value: 1.9539999999999997 + - type: precision_at_1000 + value: 0.207 + - type: mrr_at_1 + value: 60.3406 + - type: mrr_at_3 + value: 68.8564 + - type: mrr_at_5 + value: 70.51089999999999 + - type: mrr_at_10 + value: 71.3043 + - type: mrr_at_20 + value: 71.5148 + - type: mrr_at_100 + value: 71.5779 + - type: mrr_at_1000 + value: 71.5857 + - type: nauc_ndcg_at_1_max + value: 39.480900000000005 + - type: nauc_ndcg_at_1_std + value: 4.66 + - type: nauc_ndcg_at_1_diff1 + value: 43.4568 + - type: nauc_ndcg_at_3_max + value: 34.6544 + - type: nauc_ndcg_at_3_std + value: -1.7936 + - type: nauc_ndcg_at_3_diff1 + value: 39.1951 + - type: nauc_ndcg_at_5_max + value: 36.9934 + - type: nauc_ndcg_at_5_std + value: -1.427 + - type: nauc_ndcg_at_5_diff1 + value: 39.6396 + - type: nauc_ndcg_at_10_max + value: 38.9518 + - type: nauc_ndcg_at_10_std + value: 0.1574 + - type: nauc_ndcg_at_10_diff1 + value: 37.6783 + - type: nauc_ndcg_at_20_max + value: 38.5914 + - type: nauc_ndcg_at_20_std + value: 1.8135999999999999 + - type: nauc_ndcg_at_20_diff1 + value: 38.063 + - type: nauc_ndcg_at_100_max + value: 40.2409 + - type: nauc_ndcg_at_100_std + value: 5.0953 + - type: nauc_ndcg_at_100_diff1 + value: 38.5175 + - type: nauc_ndcg_at_1000_max + value: 39.9212 + - type: nauc_ndcg_at_1000_std + value: 4.5499 + - type: nauc_ndcg_at_1000_diff1 + value: 38.6193 + - type: nauc_map_at_1_max + value: 17.9005 + - type: nauc_map_at_1_std + value: -15.587699999999998 + - type: nauc_map_at_1_diff1 + value: 48.1378 + - type: nauc_map_at_3_max + value: 28.119300000000003 + - type: nauc_map_at_3_std + value: -11.3599 + - type: nauc_map_at_3_diff1 + value: 41.3327 + - type: nauc_map_at_5_max + value: 32.3026 + - type: nauc_map_at_5_std + value: -7.741499999999999 + - type: nauc_map_at_5_diff1 + value: 40.5989 + - type: nauc_map_at_10_max + value: 33.8864 + - type: nauc_map_at_10_std + value: -5.6699 + - type: nauc_map_at_10_diff1 + value: 39.586 + - type: nauc_map_at_20_max + value: 34.0193 + - type: nauc_map_at_20_std + value: -4.6238 + - type: nauc_map_at_20_diff1 + value: 39.7785 + - type: nauc_map_at_100_max + value: 34.475699999999996 + - type: nauc_map_at_100_std + value: -3.6669 + - type: nauc_map_at_100_diff1 + value: 39.8911 + - type: nauc_map_at_1000_max + value: 34.4983 + - type: nauc_map_at_1000_std + value: -3.6664000000000003 + - type: nauc_map_at_1000_diff1 + value: 39.9015 + - type: nauc_recall_at_1_max + value: 17.9005 + - type: nauc_recall_at_1_std + value: -15.587699999999998 + - type: nauc_recall_at_1_diff1 + value: 48.1378 + - type: nauc_recall_at_3_max + value: 27.0807 + - type: nauc_recall_at_3_std + value: -10.071 + - type: nauc_recall_at_3_diff1 + value: 35.7245 + - type: nauc_recall_at_5_max + value: 32.561499999999995 + - type: nauc_recall_at_5_std + value: -7.4364 + - type: nauc_recall_at_5_diff1 + value: 32.2967 + - type: nauc_recall_at_10_max + value: 36.9998 + - type: nauc_recall_at_10_std + value: -1.9453000000000003 + - type: nauc_recall_at_10_diff1 + value: 23.9665 + - type: nauc_recall_at_20_max + value: 34.0415 + - type: nauc_recall_at_20_std + value: 3.2483999999999997 + - type: nauc_recall_at_20_diff1 + value: 22.3991 + - type: nauc_recall_at_100_max + value: 52.1359 + - type: nauc_recall_at_100_std + value: 39.305299999999995 + - type: nauc_recall_at_100_diff1 + value: 17.8559 + - type: nauc_recall_at_1000_max + value: 53.5217 + - type: nauc_recall_at_1000_std + value: 78.536 + - type: nauc_recall_at_1000_diff1 + value: -24.390600000000003 + - type: nauc_precision_at_1_max + value: 39.480900000000005 + - type: nauc_precision_at_1_std + value: 4.66 + - type: nauc_precision_at_1_diff1 + value: 43.4568 + - type: nauc_precision_at_3_max + value: 38.954499999999996 + - type: nauc_precision_at_3_std + value: 21.0387 + - type: nauc_precision_at_3_diff1 + value: 4.625900000000001 + - type: nauc_precision_at_5_max + value: 38.8673 + - type: nauc_precision_at_5_std + value: 31.512800000000002 + - type: nauc_precision_at_5_diff1 + value: -4.147399999999999 + - type: nauc_precision_at_10_max + value: 32.7684 + - type: nauc_precision_at_10_std + value: 36.237700000000004 + - type: nauc_precision_at_10_diff1 + value: -13.6404 + - type: nauc_precision_at_20_max + value: 26.0982 + - type: nauc_precision_at_20_std + value: 38.5385 + - type: nauc_precision_at_20_diff1 + value: -16.3735 + - type: nauc_precision_at_100_max + value: 20.8957 + - type: nauc_precision_at_100_std + value: 42.1707 + - type: nauc_precision_at_100_diff1 + value: -18.7092 + - type: nauc_precision_at_1000_max + value: 17.1788 + - type: nauc_precision_at_1000_std + value: 39.5064 + - type: nauc_precision_at_1000_diff1 + value: -20.671400000000002 + - type: nauc_mrr_at_1_max + value: 39.480900000000005 + - type: nauc_mrr_at_1_std + value: 4.66 + - type: nauc_mrr_at_1_diff1 + value: 43.4568 + - type: nauc_mrr_at_3_max + value: 44.2708 + - type: nauc_mrr_at_3_std + value: 11.021799999999999 + - type: nauc_mrr_at_3_diff1 + value: 41.6187 + - type: nauc_mrr_at_5_max + value: 44.9277 + - type: nauc_mrr_at_5_std + value: 11.3479 + - type: nauc_mrr_at_5_diff1 + value: 41.14 + - type: nauc_mrr_at_10_max + value: 44.6467 + - type: nauc_mrr_at_10_std + value: 11.3277 + - type: nauc_mrr_at_10_diff1 + value: 40.5017 + - type: nauc_mrr_at_20_max + value: 44.298 + - type: nauc_mrr_at_20_std + value: 11.0061 + - type: nauc_mrr_at_20_diff1 + value: 40.6235 + - type: nauc_mrr_at_100_max + value: 44.2517 + - type: nauc_mrr_at_100_std + value: 10.9246 + - type: nauc_mrr_at_100_diff1 + value: 40.7234 + - type: nauc_mrr_at_1000_max + value: 44.241 + - type: nauc_mrr_at_1000_std + value: 10.9113 + - type: nauc_mrr_at_1000_diff1 + value: 40.7358 + - type: main_score + value: 68.05499999999999 + task: + type: Retrieval + - dataset: + config: de + name: MTEB MIRACLRetrieval (de) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 45.574 + - type: ndcg_at_3 + value: 41.243 + - type: ndcg_at_5 + value: 43.86 + - type: ndcg_at_10 + value: 48.123 + - type: ndcg_at_20 + value: 51.785000000000004 + - type: ndcg_at_100 + value: 56.04900000000001 + - type: ndcg_at_1000 + value: 57.979 + - type: map_at_1 + value: 20.401 + - type: map_at_3 + value: 31.308000000000003 + - type: map_at_5 + value: 35.356 + - type: map_at_10 + value: 38.24 + - type: map_at_20 + value: 39.879 + - type: map_at_100 + value: 40.979 + - type: map_at_1000 + value: 41.103 + - type: recall_at_1 + value: 20.401 + - type: recall_at_3 + value: 36.573 + - type: recall_at_5 + value: 47.495 + - type: recall_at_10 + value: 58.779 + - type: recall_at_20 + value: 69.06099999999999 + - type: recall_at_100 + value: 85.84 + - type: recall_at_1000 + value: 97.36399999999999 + - type: precision_at_1 + value: 45.574 + - type: precision_at_3 + value: 30.055 + - type: precision_at_5 + value: 23.344 + - type: precision_at_10 + value: 14.754000000000001 + - type: precision_at_20 + value: 9.033 + - type: precision_at_100 + value: 2.275 + - type: precision_at_1000 + value: 0.258 + - type: mrr_at_1 + value: 45.5738 + - type: mrr_at_3 + value: 52.18580000000001 + - type: mrr_at_5 + value: 54.5628 + - type: mrr_at_10 + value: 55.604699999999994 + - type: mrr_at_20 + value: 55.9833 + - type: mrr_at_100 + value: 56.2015 + - type: mrr_at_1000 + value: 56.2431 + - type: nauc_ndcg_at_1_max + value: 48.355 + - type: nauc_ndcg_at_1_std + value: 15.508 + - type: nauc_ndcg_at_1_diff1 + value: 42.6569 + - type: nauc_ndcg_at_3_max + value: 45.5945 + - type: nauc_ndcg_at_3_std + value: 16.6953 + - type: nauc_ndcg_at_3_diff1 + value: 38.6081 + - type: nauc_ndcg_at_5_max + value: 43.3231 + - type: nauc_ndcg_at_5_std + value: 14.394100000000002 + - type: nauc_ndcg_at_5_diff1 + value: 38.846799999999995 + - type: nauc_ndcg_at_10_max + value: 44.0599 + - type: nauc_ndcg_at_10_std + value: 16.0584 + - type: nauc_ndcg_at_10_diff1 + value: 38.2432 + - type: nauc_ndcg_at_20_max + value: 45.8588 + - type: nauc_ndcg_at_20_std + value: 17.531 + - type: nauc_ndcg_at_20_diff1 + value: 38.982099999999996 + - type: nauc_ndcg_at_100_max + value: 48.7095 + - type: nauc_ndcg_at_100_std + value: 20.7655 + - type: nauc_ndcg_at_100_diff1 + value: 39.7349 + - type: nauc_ndcg_at_1000_max + value: 48.024499999999996 + - type: nauc_ndcg_at_1000_std + value: 20.1299 + - type: nauc_ndcg_at_1000_diff1 + value: 39.8087 + - type: nauc_map_at_1_max + value: 30.0998 + - type: nauc_map_at_1_std + value: 4.7429 + - type: nauc_map_at_1_diff1 + value: 45.4045 + - type: nauc_map_at_3_max + value: 39.053399999999996 + - type: nauc_map_at_3_std + value: 10.807 + - type: nauc_map_at_3_diff1 + value: 40.8294 + - type: nauc_map_at_5_max + value: 39.204499999999996 + - type: nauc_map_at_5_std + value: 11.5165 + - type: nauc_map_at_5_diff1 + value: 38.9168 + - type: nauc_map_at_10_max + value: 41.099799999999995 + - type: nauc_map_at_10_std + value: 13.758899999999999 + - type: nauc_map_at_10_diff1 + value: 38.2256 + - type: nauc_map_at_20_max + value: 42.2131 + - type: nauc_map_at_20_std + value: 14.366000000000001 + - type: nauc_map_at_20_diff1 + value: 38.572 + - type: nauc_map_at_100_max + value: 43.0508 + - type: nauc_map_at_100_std + value: 15.060100000000002 + - type: nauc_map_at_100_diff1 + value: 38.9831 + - type: nauc_map_at_1000_max + value: 43.048700000000004 + - type: nauc_map_at_1000_std + value: 15.085999999999999 + - type: nauc_map_at_1000_diff1 + value: 38.9957 + - type: nauc_recall_at_1_max + value: 30.0998 + - type: nauc_recall_at_1_std + value: 4.7429 + - type: nauc_recall_at_1_diff1 + value: 45.4045 + - type: nauc_recall_at_3_max + value: 36.9204 + - type: nauc_recall_at_3_std + value: 11.2734 + - type: nauc_recall_at_3_diff1 + value: 37.431 + - type: nauc_recall_at_5_max + value: 33.4392 + - type: nauc_recall_at_5_std + value: 9.4283 + - type: nauc_recall_at_5_diff1 + value: 32.7815 + - type: nauc_recall_at_10_max + value: 34.427099999999996 + - type: nauc_recall_at_10_std + value: 13.147400000000001 + - type: nauc_recall_at_10_diff1 + value: 29.394199999999998 + - type: nauc_recall_at_20_max + value: 36.8459 + - type: nauc_recall_at_20_std + value: 16.1323 + - type: nauc_recall_at_20_diff1 + value: 29.9502 + - type: nauc_recall_at_100_max + value: 56.360600000000005 + - type: nauc_recall_at_100_std + value: 40.8465 + - type: nauc_recall_at_100_diff1 + value: 33.2542 + - type: nauc_recall_at_1000_max + value: 62.121 + - type: nauc_recall_at_1000_std + value: 65.4518 + - type: nauc_recall_at_1000_diff1 + value: 23.9221 + - type: nauc_precision_at_1_max + value: 48.355 + - type: nauc_precision_at_1_std + value: 15.508 + - type: nauc_precision_at_1_diff1 + value: 42.6569 + - type: nauc_precision_at_3_max + value: 46.72 + - type: nauc_precision_at_3_std + value: 21.5057 + - type: nauc_precision_at_3_diff1 + value: 23.3313 + - type: nauc_precision_at_5_max + value: 39.5888 + - type: nauc_precision_at_5_std + value: 20.930699999999998 + - type: nauc_precision_at_5_diff1 + value: 15.661900000000001 + - type: nauc_precision_at_10_max + value: 37.8371 + - type: nauc_precision_at_10_std + value: 25.2882 + - type: nauc_precision_at_10_diff1 + value: 8.7263 + - type: nauc_precision_at_20_max + value: 34.7638 + - type: nauc_precision_at_20_std + value: 25.795800000000003 + - type: nauc_precision_at_20_diff1 + value: 5.5533 + - type: nauc_precision_at_100_max + value: 31.1513 + - type: nauc_precision_at_100_std + value: 28.7441 + - type: nauc_precision_at_100_diff1 + value: -0.2107 + - type: nauc_precision_at_1000_max + value: 24.329700000000003 + - type: nauc_precision_at_1000_std + value: 27.4593 + - type: nauc_precision_at_1000_diff1 + value: -5.1174 + - type: nauc_mrr_at_1_max + value: 48.355 + - type: nauc_mrr_at_1_std + value: 15.508 + - type: nauc_mrr_at_1_diff1 + value: 42.6569 + - type: nauc_mrr_at_3_max + value: 50.1901 + - type: nauc_mrr_at_3_std + value: 17.6811 + - type: nauc_mrr_at_3_diff1 + value: 42.7492 + - type: nauc_mrr_at_5_max + value: 50.210699999999996 + - type: nauc_mrr_at_5_std + value: 17.4661 + - type: nauc_mrr_at_5_diff1 + value: 42.9336 + - type: nauc_mrr_at_10_max + value: 49.9472 + - type: nauc_mrr_at_10_std + value: 17.3815 + - type: nauc_mrr_at_10_diff1 + value: 42.4177 + - type: nauc_mrr_at_20_max + value: 49.9918 + - type: nauc_mrr_at_20_std + value: 17.7321 + - type: nauc_mrr_at_20_diff1 + value: 42.5105 + - type: nauc_mrr_at_100_max + value: 49.9862 + - type: nauc_mrr_at_100_std + value: 17.7582 + - type: nauc_mrr_at_100_diff1 + value: 42.5947 + - type: nauc_mrr_at_1000_max + value: 49.9819 + - type: nauc_mrr_at_1000_std + value: 17.7188 + - type: nauc_mrr_at_1000_diff1 + value: 42.620000000000005 + - type: main_score + value: 48.123 + task: + type: Retrieval + - dataset: + config: en + name: MTEB MIRACLRetrieval (en) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 45.556999999999995 + - type: ndcg_at_3 + value: 43.969 + - type: ndcg_at_5 + value: 45.551 + - type: ndcg_at_10 + value: 49.372 + - type: ndcg_at_20 + value: 52.86300000000001 + - type: ndcg_at_100 + value: 57.28 + - type: ndcg_at_1000 + value: 59.187 + - type: map_at_1 + value: 21.785 + - type: map_at_3 + value: 32.679 + - type: map_at_5 + value: 35.885 + - type: map_at_10 + value: 38.836 + - type: map_at_20 + value: 40.425 + - type: map_at_100 + value: 41.592 + - type: map_at_1000 + value: 41.749 + - type: recall_at_1 + value: 21.785 + - type: recall_at_3 + value: 40.403 + - type: recall_at_5 + value: 48.498999999999995 + - type: recall_at_10 + value: 59.513000000000005 + - type: recall_at_20 + value: 69.357 + - type: recall_at_100 + value: 85.785 + - type: recall_at_1000 + value: 96.041 + - type: precision_at_1 + value: 45.556999999999995 + - type: precision_at_3 + value: 30.287999999999997 + - type: precision_at_5 + value: 23.204 + - type: precision_at_10 + value: 15.006 + - type: precision_at_20 + value: 9.118 + - type: precision_at_100 + value: 2.404 + - type: precision_at_1000 + value: 0.27799999999999997 + - type: mrr_at_1 + value: 45.5569 + - type: mrr_at_3 + value: 55.4234 + - type: mrr_at_5 + value: 57.3884 + - type: mrr_at_10 + value: 58.391400000000004 + - type: mrr_at_20 + value: 58.7477 + - type: mrr_at_100 + value: 58.93620000000001 + - type: mrr_at_1000 + value: 58.949600000000004 + - type: nauc_ndcg_at_1_max + value: 34.794799999999995 + - type: nauc_ndcg_at_1_std + value: 2.102 + - type: nauc_ndcg_at_1_diff1 + value: 33.8113 + - type: nauc_ndcg_at_3_max + value: 31.6187 + - type: nauc_ndcg_at_3_std + value: -1.3106 + - type: nauc_ndcg_at_3_diff1 + value: 28.5676 + - type: nauc_ndcg_at_5_max + value: 30.4962 + - type: nauc_ndcg_at_5_std + value: -1.016 + - type: nauc_ndcg_at_5_diff1 + value: 28.0032 + - type: nauc_ndcg_at_10_max + value: 29.460900000000002 + - type: nauc_ndcg_at_10_std + value: -0.6328 + - type: nauc_ndcg_at_10_diff1 + value: 26.351000000000003 + - type: nauc_ndcg_at_20_max + value: 31.443900000000003 + - type: nauc_ndcg_at_20_std + value: 1.1067 + - type: nauc_ndcg_at_20_diff1 + value: 26.2068 + - type: nauc_ndcg_at_100_max + value: 34.273199999999996 + - type: nauc_ndcg_at_100_std + value: 5.1303 + - type: nauc_ndcg_at_100_diff1 + value: 26.4772 + - type: nauc_ndcg_at_1000_max + value: 34.1663 + - type: nauc_ndcg_at_1000_std + value: 5.1834999999999996 + - type: nauc_ndcg_at_1000_diff1 + value: 26.6768 + - type: nauc_map_at_1_max + value: 23.6327 + - type: nauc_map_at_1_std + value: -6.3777 + - type: nauc_map_at_1_diff1 + value: 32.028800000000004 + - type: nauc_map_at_3_max + value: 27.869300000000003 + - type: nauc_map_at_3_std + value: -5.9788 + - type: nauc_map_at_3_diff1 + value: 29.8636 + - type: nauc_map_at_5_max + value: 28.6043 + - type: nauc_map_at_5_std + value: -4.4539 + - type: nauc_map_at_5_diff1 + value: 29.044999999999998 + - type: nauc_map_at_10_max + value: 29.065600000000003 + - type: nauc_map_at_10_std + value: -3.2986 + - type: nauc_map_at_10_diff1 + value: 27.8952 + - type: nauc_map_at_20_max + value: 30.191200000000002 + - type: nauc_map_at_20_std + value: -2.4181999999999997 + - type: nauc_map_at_20_diff1 + value: 27.973399999999998 + - type: nauc_map_at_100_max + value: 31.0841 + - type: nauc_map_at_100_std + value: -1.1223 + - type: nauc_map_at_100_diff1 + value: 28.089199999999998 + - type: nauc_map_at_1000_max + value: 31.114399999999996 + - type: nauc_map_at_1000_std + value: -1.0668 + - type: nauc_map_at_1000_diff1 + value: 28.098 + - type: nauc_recall_at_1_max + value: 23.6327 + - type: nauc_recall_at_1_std + value: -6.3777 + - type: nauc_recall_at_1_diff1 + value: 32.028800000000004 + - type: nauc_recall_at_3_max + value: 20.9084 + - type: nauc_recall_at_3_std + value: -7.3713 + - type: nauc_recall_at_3_diff1 + value: 23.488300000000002 + - type: nauc_recall_at_5_max + value: 20.4249 + - type: nauc_recall_at_5_std + value: -3.8598 + - type: nauc_recall_at_5_diff1 + value: 20.935200000000002 + - type: nauc_recall_at_10_max + value: 17.5405 + - type: nauc_recall_at_10_std + value: -3.5011 + - type: nauc_recall_at_10_diff1 + value: 16.9646 + - type: nauc_recall_at_20_max + value: 20.6496 + - type: nauc_recall_at_20_std + value: 0.1168 + - type: nauc_recall_at_20_diff1 + value: 14.2125 + - type: nauc_recall_at_100_max + value: 31.916099999999997 + - type: nauc_recall_at_100_std + value: 20.2048 + - type: nauc_recall_at_100_diff1 + value: 9.3709 + - type: nauc_recall_at_1000_max + value: 46.2569 + - type: nauc_recall_at_1000_std + value: 55.2292 + - type: nauc_recall_at_1000_diff1 + value: -0.2909 + - type: nauc_precision_at_1_max + value: 34.794799999999995 + - type: nauc_precision_at_1_std + value: 2.102 + - type: nauc_precision_at_1_diff1 + value: 33.8113 + - type: nauc_precision_at_3_max + value: 31.221700000000002 + - type: nauc_precision_at_3_std + value: 7.513 + - type: nauc_precision_at_3_diff1 + value: 15.9311 + - type: nauc_precision_at_5_max + value: 28.5241 + - type: nauc_precision_at_5_std + value: 12.2286 + - type: nauc_precision_at_5_diff1 + value: 9.5435 + - type: nauc_precision_at_10_max + value: 24.3663 + - type: nauc_precision_at_10_std + value: 15.867700000000001 + - type: nauc_precision_at_10_diff1 + value: 2.396 + - type: nauc_precision_at_20_max + value: 22.322300000000002 + - type: nauc_precision_at_20_std + value: 18.3505 + - type: nauc_precision_at_20_diff1 + value: 0.0719 + - type: nauc_precision_at_100_max + value: 18.8029 + - type: nauc_precision_at_100_std + value: 24.728 + - type: nauc_precision_at_100_diff1 + value: -4.0887 + - type: nauc_precision_at_1000_max + value: 12.315800000000001 + - type: nauc_precision_at_1000_std + value: 20.9058 + - type: nauc_precision_at_1000_diff1 + value: -6.4069 + - type: nauc_mrr_at_1_max + value: 34.794799999999995 + - type: nauc_mrr_at_1_std + value: 2.102 + - type: nauc_mrr_at_1_diff1 + value: 33.8113 + - type: nauc_mrr_at_3_max + value: 33.3929 + - type: nauc_mrr_at_3_std + value: 3.4512 + - type: nauc_mrr_at_3_diff1 + value: 29.718 + - type: nauc_mrr_at_5_max + value: 34.586 + - type: nauc_mrr_at_5_std + value: 5.4722 + - type: nauc_mrr_at_5_diff1 + value: 30.0744 + - type: nauc_mrr_at_10_max + value: 34.3898 + - type: nauc_mrr_at_10_std + value: 4.854 + - type: nauc_mrr_at_10_diff1 + value: 29.979 + - type: nauc_mrr_at_20_max + value: 34.516000000000005 + - type: nauc_mrr_at_20_std + value: 4.9616 + - type: nauc_mrr_at_20_diff1 + value: 29.907899999999998 + - type: nauc_mrr_at_100_max + value: 34.515499999999996 + - type: nauc_mrr_at_100_std + value: 4.8578 + - type: nauc_mrr_at_100_diff1 + value: 29.997 + - type: nauc_mrr_at_1000_max + value: 34.5046 + - type: nauc_mrr_at_1000_std + value: 4.8536 + - type: nauc_mrr_at_1000_diff1 + value: 30.0019 + - type: main_score + value: 49.372 + task: + type: Retrieval + - dataset: + config: es + name: MTEB MIRACLRetrieval (es) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 55.71 + - type: ndcg_at_3 + value: 47.981 + - type: ndcg_at_5 + value: 46.583999999999996 + - type: ndcg_at_10 + value: 49.688 + - type: ndcg_at_20 + value: 54.437999999999995 + - type: ndcg_at_100 + value: 60.492999999999995 + - type: ndcg_at_1000 + value: 62.922 + - type: map_at_1 + value: 16.38 + - type: map_at_3 + value: 27.137 + - type: map_at_5 + value: 31.81 + - type: map_at_10 + value: 36.986999999999995 + - type: map_at_20 + value: 39.749 + - type: map_at_100 + value: 41.69 + - type: map_at_1000 + value: 41.924 + - type: recall_at_1 + value: 16.38 + - type: recall_at_3 + value: 31.502999999999997 + - type: recall_at_5 + value: 40.355999999999995 + - type: recall_at_10 + value: 54.155 + - type: recall_at_20 + value: 65.32900000000001 + - type: recall_at_100 + value: 85.136 + - type: recall_at_1000 + value: 96.951 + - type: precision_at_1 + value: 55.71 + - type: precision_at_3 + value: 39.969 + - type: precision_at_5 + value: 32.469 + - type: precision_at_10 + value: 23.071 + - type: precision_at_20 + value: 14.482999999999999 + - type: precision_at_100 + value: 3.8920000000000003 + - type: precision_at_1000 + value: 0.44799999999999995 + - type: mrr_at_1 + value: 55.709900000000005 + - type: mrr_at_3 + value: 63.9146 + - type: mrr_at_5 + value: 65.4192 + - type: mrr_at_10 + value: 66.4602 + - type: mrr_at_20 + value: 66.71249999999999 + - type: mrr_at_100 + value: 66.8844 + - type: mrr_at_1000 + value: 66.893 + - type: nauc_ndcg_at_1_max + value: 39.4623 + - type: nauc_ndcg_at_1_std + value: 18.2237 + - type: nauc_ndcg_at_1_diff1 + value: 34.3382 + - type: nauc_ndcg_at_3_max + value: 33.3518 + - type: nauc_ndcg_at_3_std + value: 14.2885 + - type: nauc_ndcg_at_3_diff1 + value: 22.4965 + - type: nauc_ndcg_at_5_max + value: 31.5822 + - type: nauc_ndcg_at_5_std + value: 10.4064 + - type: nauc_ndcg_at_5_diff1 + value: 24.4417 + - type: nauc_ndcg_at_10_max + value: 33.4838 + - type: nauc_ndcg_at_10_std + value: 11.5351 + - type: nauc_ndcg_at_10_diff1 + value: 27.1137 + - type: nauc_ndcg_at_20_max + value: 38.831700000000005 + - type: nauc_ndcg_at_20_std + value: 18.784 + - type: nauc_ndcg_at_20_diff1 + value: 27.408700000000003 + - type: nauc_ndcg_at_100_max + value: 42.8785 + - type: nauc_ndcg_at_100_std + value: 24.596 + - type: nauc_ndcg_at_100_diff1 + value: 25.8252 + - type: nauc_ndcg_at_1000_max + value: 42.023500000000006 + - type: nauc_ndcg_at_1000_std + value: 23.2727 + - type: nauc_ndcg_at_1000_diff1 + value: 24.8455 + - type: nauc_map_at_1_max + value: 10.5243 + - type: nauc_map_at_1_std + value: -10.143699999999999 + - type: nauc_map_at_1_diff1 + value: 32.2699 + - type: nauc_map_at_3_max + value: 16.902900000000002 + - type: nauc_map_at_3_std + value: -5.6548 + - type: nauc_map_at_3_diff1 + value: 26.238699999999998 + - type: nauc_map_at_5_max + value: 21.4475 + - type: nauc_map_at_5_std + value: -2.1950000000000003 + - type: nauc_map_at_5_diff1 + value: 25.2077 + - type: nauc_map_at_10_max + value: 27.2231 + - type: nauc_map_at_10_std + value: 3.9522000000000004 + - type: nauc_map_at_10_diff1 + value: 26.0175 + - type: nauc_map_at_20_max + value: 30.8106 + - type: nauc_map_at_20_std + value: 8.9534 + - type: nauc_map_at_20_diff1 + value: 25.8477 + - type: nauc_map_at_100_max + value: 32.5864 + - type: nauc_map_at_100_std + value: 11.2878 + - type: nauc_map_at_100_diff1 + value: 25.3496 + - type: nauc_map_at_1000_max + value: 32.573 + - type: nauc_map_at_1000_std + value: 11.2812 + - type: nauc_map_at_1000_diff1 + value: 25.2334 + - type: nauc_recall_at_1_max + value: 10.5243 + - type: nauc_recall_at_1_std + value: -10.143699999999999 + - type: nauc_recall_at_1_diff1 + value: 32.2699 + - type: nauc_recall_at_3_max + value: 12.1019 + - type: nauc_recall_at_3_std + value: -8.2304 + - type: nauc_recall_at_3_diff1 + value: 22.9436 + - type: nauc_recall_at_5_max + value: 15.0438 + - type: nauc_recall_at_5_std + value: -6.216200000000001 + - type: nauc_recall_at_5_diff1 + value: 21.5158 + - type: nauc_recall_at_10_max + value: 22.825100000000003 + - type: nauc_recall_at_10_std + value: 4.994400000000001 + - type: nauc_recall_at_10_diff1 + value: 22.4346 + - type: nauc_recall_at_20_max + value: 33.1395 + - type: nauc_recall_at_20_std + value: 19.5456 + - type: nauc_recall_at_20_diff1 + value: 24.0575 + - type: nauc_recall_at_100_max + value: 50.0911 + - type: nauc_recall_at_100_std + value: 45.542300000000004 + - type: nauc_recall_at_100_diff1 + value: 19.9322 + - type: nauc_recall_at_1000_max + value: 73.2055 + - type: nauc_recall_at_1000_std + value: 74.8121 + - type: nauc_recall_at_1000_diff1 + value: 6.7021999999999995 + - type: nauc_precision_at_1_max + value: 39.4623 + - type: nauc_precision_at_1_std + value: 18.2237 + - type: nauc_precision_at_1_diff1 + value: 34.3382 + - type: nauc_precision_at_3_max + value: 37.2684 + - type: nauc_precision_at_3_std + value: 24.1559 + - type: nauc_precision_at_3_diff1 + value: 10.6349 + - type: nauc_precision_at_5_max + value: 37.9483 + - type: nauc_precision_at_5_std + value: 26.973000000000003 + - type: nauc_precision_at_5_diff1 + value: 6.722499999999999 + - type: nauc_precision_at_10_max + value: 41.4223 + - type: nauc_precision_at_10_std + value: 35.661100000000005 + - type: nauc_precision_at_10_diff1 + value: 3.8463 + - type: nauc_precision_at_20_max + value: 41.917300000000004 + - type: nauc_precision_at_20_std + value: 42.0563 + - type: nauc_precision_at_20_diff1 + value: 0.4484 + - type: nauc_precision_at_100_max + value: 37.4895 + - type: nauc_precision_at_100_std + value: 45.1734 + - type: nauc_precision_at_100_diff1 + value: -7.4965 + - type: nauc_precision_at_1000_max + value: 27.853299999999997 + - type: nauc_precision_at_1000_std + value: 36.997 + - type: nauc_precision_at_1000_diff1 + value: -13.5956 + - type: nauc_mrr_at_1_max + value: 39.4623 + - type: nauc_mrr_at_1_std + value: 18.2237 + - type: nauc_mrr_at_1_diff1 + value: 34.3382 + - type: nauc_mrr_at_3_max + value: 43.2341 + - type: nauc_mrr_at_3_std + value: 22.287599999999998 + - type: nauc_mrr_at_3_diff1 + value: 32.1338 + - type: nauc_mrr_at_5_max + value: 43.1729 + - type: nauc_mrr_at_5_std + value: 21.9232 + - type: nauc_mrr_at_5_diff1 + value: 32.0241 + - type: nauc_mrr_at_10_max + value: 43.8014 + - type: nauc_mrr_at_10_std + value: 23.1591 + - type: nauc_mrr_at_10_diff1 + value: 31.898100000000003 + - type: nauc_mrr_at_20_max + value: 43.7825 + - type: nauc_mrr_at_20_std + value: 23.1845 + - type: nauc_mrr_at_20_diff1 + value: 32.2338 + - type: nauc_mrr_at_100_max + value: 43.6665 + - type: nauc_mrr_at_100_std + value: 23.0026 + - type: nauc_mrr_at_100_diff1 + value: 32.177299999999995 + - type: nauc_mrr_at_1000_max + value: 43.6579 + - type: nauc_mrr_at_1000_std + value: 22.986500000000003 + - type: nauc_mrr_at_1000_diff1 + value: 32.1927 + - type: main_score + value: 49.688 + task: + type: Retrieval + - dataset: + config: fa + name: MTEB MIRACLRetrieval (fa) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 39.873 + - type: ndcg_at_3 + value: 42.738 + - type: ndcg_at_5 + value: 45.843 + - type: ndcg_at_10 + value: 50.226000000000006 + - type: ndcg_at_20 + value: 52.92 + - type: ndcg_at_100 + value: 56.516999999999996 + - type: ndcg_at_1000 + value: 57.967 + - type: map_at_1 + value: 25.369000000000003 + - type: map_at_3 + value: 35.791000000000004 + - type: map_at_5 + value: 39.027 + - type: map_at_10 + value: 41.759 + - type: map_at_20 + value: 42.899 + - type: map_at_100 + value: 43.637 + - type: map_at_1000 + value: 43.734 + - type: recall_at_1 + value: 25.369000000000003 + - type: recall_at_3 + value: 43.808 + - type: recall_at_5 + value: 52.378 + - type: recall_at_10 + value: 63.775999999999996 + - type: recall_at_20 + value: 72.099 + - type: recall_at_100 + value: 87.68599999999999 + - type: recall_at_1000 + value: 96.71 + - type: precision_at_1 + value: 39.873 + - type: precision_at_3 + value: 25.580000000000002 + - type: precision_at_5 + value: 19.367 + - type: precision_at_10 + value: 12.437 + - type: precision_at_20 + value: 7.247000000000001 + - type: precision_at_100 + value: 1.807 + - type: precision_at_1000 + value: 0.202 + - type: mrr_at_1 + value: 39.8734 + - type: mrr_at_3 + value: 49.1297 + - type: mrr_at_5 + value: 50.8703 + - type: mrr_at_10 + value: 52.0393 + - type: mrr_at_20 + value: 52.428 + - type: mrr_at_100 + value: 52.7259 + - type: mrr_at_1000 + value: 52.7512 + - type: nauc_ndcg_at_1_max + value: 37.2005 + - type: nauc_ndcg_at_1_std + value: 7.2856000000000005 + - type: nauc_ndcg_at_1_diff1 + value: 24.3391 + - type: nauc_ndcg_at_3_max + value: 34.9919 + - type: nauc_ndcg_at_3_std + value: 4.1377 + - type: nauc_ndcg_at_3_diff1 + value: 22.7251 + - type: nauc_ndcg_at_5_max + value: 35.3802 + - type: nauc_ndcg_at_5_std + value: 5.1718 + - type: nauc_ndcg_at_5_diff1 + value: 20.7966 + - type: nauc_ndcg_at_10_max + value: 37.5244 + - type: nauc_ndcg_at_10_std + value: 8.4159 + - type: nauc_ndcg_at_10_diff1 + value: 20.3825 + - type: nauc_ndcg_at_20_max + value: 39.457 + - type: nauc_ndcg_at_20_std + value: 10.9359 + - type: nauc_ndcg_at_20_diff1 + value: 20.1633 + - type: nauc_ndcg_at_100_max + value: 40.605799999999995 + - type: nauc_ndcg_at_100_std + value: 12.8063 + - type: nauc_ndcg_at_100_diff1 + value: 20.1186 + - type: nauc_ndcg_at_1000_max + value: 39.6952 + - type: nauc_ndcg_at_1000_std + value: 12.0795 + - type: nauc_ndcg_at_1000_diff1 + value: 20.1048 + - type: nauc_map_at_1_max + value: 22.758200000000002 + - type: nauc_map_at_1_std + value: -4.4208 + - type: nauc_map_at_1_diff1 + value: 32.8042 + - type: nauc_map_at_3_max + value: 29.5871 + - type: nauc_map_at_3_std + value: -1.0369 + - type: nauc_map_at_3_diff1 + value: 26.7399 + - type: nauc_map_at_5_max + value: 31.630799999999997 + - type: nauc_map_at_5_std + value: 1.133 + - type: nauc_map_at_5_diff1 + value: 23.9264 + - type: nauc_map_at_10_max + value: 33.5866 + - type: nauc_map_at_10_std + value: 3.8602999999999996 + - type: nauc_map_at_10_diff1 + value: 23.0431 + - type: nauc_map_at_20_max + value: 34.7099 + - type: nauc_map_at_20_std + value: 5.2187 + - type: nauc_map_at_20_diff1 + value: 22.751099999999997 + - type: nauc_map_at_100_max + value: 35.0549 + - type: nauc_map_at_100_std + value: 5.7357 + - type: nauc_map_at_100_diff1 + value: 22.7261 + - type: nauc_map_at_1000_max + value: 35.02 + - type: nauc_map_at_1000_std + value: 5.7542 + - type: nauc_map_at_1000_diff1 + value: 22.717000000000002 + - type: nauc_recall_at_1_max + value: 22.758200000000002 + - type: nauc_recall_at_1_std + value: -4.4208 + - type: nauc_recall_at_1_diff1 + value: 32.8042 + - type: nauc_recall_at_3_max + value: 29.2098 + - type: nauc_recall_at_3_std + value: 0.1884 + - type: nauc_recall_at_3_diff1 + value: 21.9167 + - type: nauc_recall_at_5_max + value: 30.634099999999997 + - type: nauc_recall_at_5_std + value: 2.9632 + - type: nauc_recall_at_5_diff1 + value: 15.8588 + - type: nauc_recall_at_10_max + value: 34.958 + - type: nauc_recall_at_10_std + value: 10.6769 + - type: nauc_recall_at_10_diff1 + value: 13.9022 + - type: nauc_recall_at_20_max + value: 40.5569 + - type: nauc_recall_at_20_std + value: 18.1782 + - type: nauc_recall_at_20_diff1 + value: 13.4488 + - type: nauc_recall_at_100_max + value: 54.6126 + - type: nauc_recall_at_100_std + value: 39.507999999999996 + - type: nauc_recall_at_100_diff1 + value: 10.122 + - type: nauc_recall_at_1000_max + value: 64.1019 + - type: nauc_recall_at_1000_std + value: 65.3022 + - type: nauc_recall_at_1000_diff1 + value: -0.9008 + - type: nauc_precision_at_1_max + value: 37.2005 + - type: nauc_precision_at_1_std + value: 7.2856000000000005 + - type: nauc_precision_at_1_diff1 + value: 24.3391 + - type: nauc_precision_at_3_max + value: 40.8492 + - type: nauc_precision_at_3_std + value: 14.955099999999998 + - type: nauc_precision_at_3_diff1 + value: 5.8083 + - type: nauc_precision_at_5_max + value: 37.6411 + - type: nauc_precision_at_5_std + value: 20.1371 + - type: nauc_precision_at_5_diff1 + value: -4.7182 + - type: nauc_precision_at_10_max + value: 35.9345 + - type: nauc_precision_at_10_std + value: 27.593899999999998 + - type: nauc_precision_at_10_diff1 + value: -9.1429 + - type: nauc_precision_at_20_max + value: 33.7364 + - type: nauc_precision_at_20_std + value: 31.8223 + - type: nauc_precision_at_20_diff1 + value: -11.98 + - type: nauc_precision_at_100_max + value: 25.7037 + - type: nauc_precision_at_100_std + value: 32.6954 + - type: nauc_precision_at_100_diff1 + value: -15.2838 + - type: nauc_precision_at_1000_max + value: 16.6881 + - type: nauc_precision_at_1000_std + value: 27.787200000000002 + - type: nauc_precision_at_1000_diff1 + value: -16.964000000000002 + - type: nauc_mrr_at_1_max + value: 37.2005 + - type: nauc_mrr_at_1_std + value: 7.2856000000000005 + - type: nauc_mrr_at_1_diff1 + value: 24.3391 + - type: nauc_mrr_at_3_max + value: 40.9867 + - type: nauc_mrr_at_3_std + value: 10.7794 + - type: nauc_mrr_at_3_diff1 + value: 21.0522 + - type: nauc_mrr_at_5_max + value: 40.7712 + - type: nauc_mrr_at_5_std + value: 11.2036 + - type: nauc_mrr_at_5_diff1 + value: 20.3769 + - type: nauc_mrr_at_10_max + value: 40.8976 + - type: nauc_mrr_at_10_std + value: 11.7276 + - type: nauc_mrr_at_10_diff1 + value: 20.261699999999998 + - type: nauc_mrr_at_20_max + value: 40.8283 + - type: nauc_mrr_at_20_std + value: 11.6606 + - type: nauc_mrr_at_20_diff1 + value: 20.430300000000003 + - type: nauc_mrr_at_100_max + value: 40.9123 + - type: nauc_mrr_at_100_std + value: 11.6937 + - type: nauc_mrr_at_100_diff1 + value: 20.4759 + - type: nauc_mrr_at_1000_max + value: 40.895399999999995 + - type: nauc_mrr_at_1000_std + value: 11.6648 + - type: nauc_mrr_at_1000_diff1 + value: 20.4831 + - type: main_score + value: 50.226000000000006 + task: + type: Retrieval + - dataset: + config: fi + name: MTEB MIRACLRetrieval (fi) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 60.818000000000005 + - type: ndcg_at_3 + value: 60.06 + - type: ndcg_at_5 + value: 63.842 + - type: ndcg_at_10 + value: 67.46 + - type: ndcg_at_20 + value: 69.692 + - type: ndcg_at_100 + value: 71.516 + - type: ndcg_at_1000 + value: 72.18 + - type: map_at_1 + value: 39.263999999999996 + - type: map_at_3 + value: 53.723 + - type: map_at_5 + value: 57.118 + - type: map_at_10 + value: 59.394000000000005 + - type: map_at_20 + value: 60.339 + - type: map_at_100 + value: 60.739 + - type: map_at_1000 + value: 60.782000000000004 + - type: recall_at_1 + value: 39.263999999999996 + - type: recall_at_3 + value: 61.05500000000001 + - type: recall_at_5 + value: 69.774 + - type: recall_at_10 + value: 78.577 + - type: recall_at_20 + value: 85.435 + - type: recall_at_100 + value: 93.291 + - type: recall_at_1000 + value: 97.493 + - type: precision_at_1 + value: 60.818000000000005 + - type: precision_at_3 + value: 35.064 + - type: precision_at_5 + value: 24.815 + - type: precision_at_10 + value: 14.445 + - type: precision_at_20 + value: 8.049000000000001 + - type: precision_at_100 + value: 1.7819999999999998 + - type: precision_at_1000 + value: 0.187 + - type: mrr_at_1 + value: 60.8183 + - type: mrr_at_3 + value: 68.7516 + - type: mrr_at_5 + value: 70.1678 + - type: mrr_at_10 + value: 70.85040000000001 + - type: mrr_at_20 + value: 71.1314 + - type: mrr_at_100 + value: 71.2271 + - type: mrr_at_1000 + value: 71.2334 + - type: nauc_ndcg_at_1_max + value: 39.623000000000005 + - type: nauc_ndcg_at_1_std + value: -0.6057 + - type: nauc_ndcg_at_1_diff1 + value: 50.2688 + - type: nauc_ndcg_at_3_max + value: 36.2982 + - type: nauc_ndcg_at_3_std + value: -0.4931 + - type: nauc_ndcg_at_3_diff1 + value: 41.5229 + - type: nauc_ndcg_at_5_max + value: 37.1813 + - type: nauc_ndcg_at_5_std + value: -1.1114000000000002 + - type: nauc_ndcg_at_5_diff1 + value: 41.429700000000004 + - type: nauc_ndcg_at_10_max + value: 39.3656 + - type: nauc_ndcg_at_10_std + value: 0.2202 + - type: nauc_ndcg_at_10_diff1 + value: 41.4453 + - type: nauc_ndcg_at_20_max + value: 40.186 + - type: nauc_ndcg_at_20_std + value: 2.8166 + - type: nauc_ndcg_at_20_diff1 + value: 41.0657 + - type: nauc_ndcg_at_100_max + value: 40.2423 + - type: nauc_ndcg_at_100_std + value: 4.5445 + - type: nauc_ndcg_at_100_diff1 + value: 42.1274 + - type: nauc_ndcg_at_1000_max + value: 39.821200000000005 + - type: nauc_ndcg_at_1000_std + value: 3.71 + - type: nauc_ndcg_at_1000_diff1 + value: 42.2532 + - type: nauc_map_at_1_max + value: 25.539 + - type: nauc_map_at_1_std + value: -7.6318 + - type: nauc_map_at_1_diff1 + value: 47.2875 + - type: nauc_map_at_3_max + value: 33.5096 + - type: nauc_map_at_3_std + value: -3.4685 + - type: nauc_map_at_3_diff1 + value: 41.2351 + - type: nauc_map_at_5_max + value: 35.0144 + - type: nauc_map_at_5_std + value: -2.9198999999999997 + - type: nauc_map_at_5_diff1 + value: 40.892 + - type: nauc_map_at_10_max + value: 36.4497 + - type: nauc_map_at_10_std + value: -1.8148999999999997 + - type: nauc_map_at_10_diff1 + value: 40.823100000000004 + - type: nauc_map_at_20_max + value: 36.863 + - type: nauc_map_at_20_std + value: -0.7572 + - type: nauc_map_at_20_diff1 + value: 40.6285 + - type: nauc_map_at_100_max + value: 36.882 + - type: nauc_map_at_100_std + value: -0.40850000000000003 + - type: nauc_map_at_100_diff1 + value: 40.844500000000004 + - type: nauc_map_at_1000_max + value: 36.8736 + - type: nauc_map_at_1000_std + value: -0.4359 + - type: nauc_map_at_1000_diff1 + value: 40.8569 + - type: nauc_recall_at_1_max + value: 25.539 + - type: nauc_recall_at_1_std + value: -7.6318 + - type: nauc_recall_at_1_diff1 + value: 47.2875 + - type: nauc_recall_at_3_max + value: 32.7716 + - type: nauc_recall_at_3_std + value: -1.6856 + - type: nauc_recall_at_3_diff1 + value: 36.4533 + - type: nauc_recall_at_5_max + value: 33.5681 + - type: nauc_recall_at_5_std + value: -2.4453 + - type: nauc_recall_at_5_diff1 + value: 33.8472 + - type: nauc_recall_at_10_max + value: 39.5319 + - type: nauc_recall_at_10_std + value: 0.6228 + - type: nauc_recall_at_10_diff1 + value: 31.935200000000002 + - type: nauc_recall_at_20_max + value: 44.3495 + - type: nauc_recall_at_20_std + value: 12.5445 + - type: nauc_recall_at_20_diff1 + value: 27.6315 + - type: nauc_recall_at_100_max + value: 53.924499999999995 + - type: nauc_recall_at_100_std + value: 44.5927 + - type: nauc_recall_at_100_diff1 + value: 32.2776 + - type: nauc_recall_at_1000_max + value: 59.7088 + - type: nauc_recall_at_1000_std + value: 61.6974 + - type: nauc_recall_at_1000_diff1 + value: 28.367700000000003 + - type: nauc_precision_at_1_max + value: 39.623000000000005 + - type: nauc_precision_at_1_std + value: -0.6057 + - type: nauc_precision_at_1_diff1 + value: 50.2688 + - type: nauc_precision_at_3_max + value: 29.5187 + - type: nauc_precision_at_3_std + value: 11.1305 + - type: nauc_precision_at_3_diff1 + value: 11.674 + - type: nauc_precision_at_5_max + value: 25.5889 + - type: nauc_precision_at_5_std + value: 13.4716 + - type: nauc_precision_at_5_diff1 + value: 3.2894 + - type: nauc_precision_at_10_max + value: 21.2446 + - type: nauc_precision_at_10_std + value: 15.7787 + - type: nauc_precision_at_10_diff1 + value: -4.0968 + - type: nauc_precision_at_20_max + value: 15.9944 + - type: nauc_precision_at_20_std + value: 22.4212 + - type: nauc_precision_at_20_diff1 + value: -11.3771 + - type: nauc_precision_at_100_max + value: 8.592600000000001 + - type: nauc_precision_at_100_std + value: 26.4342 + - type: nauc_precision_at_100_diff1 + value: -15.402 + - type: nauc_precision_at_1000_max + value: 2.8388 + - type: nauc_precision_at_1000_std + value: 23.2317 + - type: nauc_precision_at_1000_diff1 + value: -19.1173 + - type: nauc_mrr_at_1_max + value: 39.623000000000005 + - type: nauc_mrr_at_1_std + value: -0.6057 + - type: nauc_mrr_at_1_diff1 + value: 50.2688 + - type: nauc_mrr_at_3_max + value: 41.694199999999995 + - type: nauc_mrr_at_3_std + value: 2.5751 + - type: nauc_mrr_at_3_diff1 + value: 48.6111 + - type: nauc_mrr_at_5_max + value: 41.5674 + - type: nauc_mrr_at_5_std + value: 2.7312 + - type: nauc_mrr_at_5_diff1 + value: 48.6988 + - type: nauc_mrr_at_10_max + value: 41.7364 + - type: nauc_mrr_at_10_std + value: 2.5787 + - type: nauc_mrr_at_10_diff1 + value: 48.5842 + - type: nauc_mrr_at_20_max + value: 41.7509 + - type: nauc_mrr_at_20_std + value: 2.6837 + - type: nauc_mrr_at_20_diff1 + value: 48.7196 + - type: nauc_mrr_at_100_max + value: 41.6895 + - type: nauc_mrr_at_100_std + value: 2.6545 + - type: nauc_mrr_at_100_diff1 + value: 48.7483 + - type: nauc_mrr_at_1000_max + value: 41.6849 + - type: nauc_mrr_at_1000_std + value: 2.6379 + - type: nauc_mrr_at_1000_diff1 + value: 48.753600000000006 + - type: main_score + value: 67.46 + task: + type: Retrieval + - dataset: + config: fr + name: MTEB MIRACLRetrieval (fr) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 39.65 + - type: ndcg_at_3 + value: 39.843 + - type: ndcg_at_5 + value: 44.416 + - type: ndcg_at_10 + value: 49.891000000000005 + - type: ndcg_at_20 + value: 53.163000000000004 + - type: ndcg_at_100 + value: 56.492 + - type: ndcg_at_1000 + value: 57.837 + - type: map_at_1 + value: 22.644000000000002 + - type: map_at_3 + value: 33.021 + - type: map_at_5 + value: 36.958 + - type: map_at_10 + value: 39.967999999999996 + - type: map_at_20 + value: 41.298 + - type: map_at_100 + value: 42.03 + - type: map_at_1000 + value: 42.119 + - type: recall_at_1 + value: 22.644000000000002 + - type: recall_at_3 + value: 39.798 + - type: recall_at_5 + value: 51.001 + - type: recall_at_10 + value: 65.169 + - type: recall_at_20 + value: 75.33800000000001 + - type: recall_at_100 + value: 89.786 + - type: recall_at_1000 + value: 98.08099999999999 + - type: precision_at_1 + value: 39.65 + - type: precision_at_3 + value: 25.656000000000002 + - type: precision_at_5 + value: 20.175 + - type: precision_at_10 + value: 13.120000000000001 + - type: precision_at_20 + value: 7.7410000000000005 + - type: precision_at_100 + value: 1.883 + - type: precision_at_1000 + value: 0.208 + - type: mrr_at_1 + value: 39.6501 + - type: mrr_at_3 + value: 48.7366 + - type: mrr_at_5 + value: 50.9961 + - type: mrr_at_10 + value: 52.659 + - type: mrr_at_20 + value: 53.0856 + - type: mrr_at_100 + value: 53.273199999999996 + - type: mrr_at_1000 + value: 53.2931 + - type: nauc_ndcg_at_1_max + value: 29.1135 + - type: nauc_ndcg_at_1_std + value: 13.9561 + - type: nauc_ndcg_at_1_diff1 + value: 28.410400000000003 + - type: nauc_ndcg_at_3_max + value: 29.0117 + - type: nauc_ndcg_at_3_std + value: 15.655 + - type: nauc_ndcg_at_3_diff1 + value: 19.7043 + - type: nauc_ndcg_at_5_max + value: 31.3257 + - type: nauc_ndcg_at_5_std + value: 17.4096 + - type: nauc_ndcg_at_5_diff1 + value: 20.5295 + - type: nauc_ndcg_at_10_max + value: 33.244 + - type: nauc_ndcg_at_10_std + value: 18.8436 + - type: nauc_ndcg_at_10_diff1 + value: 17.9986 + - type: nauc_ndcg_at_20_max + value: 35.0697 + - type: nauc_ndcg_at_20_std + value: 19.84 + - type: nauc_ndcg_at_20_diff1 + value: 19.611600000000003 + - type: nauc_ndcg_at_100_max + value: 34.7837 + - type: nauc_ndcg_at_100_std + value: 22.2762 + - type: nauc_ndcg_at_100_diff1 + value: 19.3138 + - type: nauc_ndcg_at_1000_max + value: 34.4487 + - type: nauc_ndcg_at_1000_std + value: 20.8402 + - type: nauc_ndcg_at_1000_diff1 + value: 20.2691 + - type: nauc_map_at_1_max + value: 20.247200000000003 + - type: nauc_map_at_1_std + value: 8.8046 + - type: nauc_map_at_1_diff1 + value: 27.227600000000002 + - type: nauc_map_at_3_max + value: 26.7076 + - type: nauc_map_at_3_std + value: 13.7464 + - type: nauc_map_at_3_diff1 + value: 21.1266 + - type: nauc_map_at_5_max + value: 28.777399999999997 + - type: nauc_map_at_5_std + value: 15.348400000000002 + - type: nauc_map_at_5_diff1 + value: 21.4282 + - type: nauc_map_at_10_max + value: 29.907600000000002 + - type: nauc_map_at_10_std + value: 16.3636 + - type: nauc_map_at_10_diff1 + value: 20.1957 + - type: nauc_map_at_20_max + value: 30.864399999999996 + - type: nauc_map_at_20_std + value: 16.936999999999998 + - type: nauc_map_at_20_diff1 + value: 20.8871 + - type: nauc_map_at_100_max + value: 30.998900000000003 + - type: nauc_map_at_100_std + value: 17.673 + - type: nauc_map_at_100_diff1 + value: 20.7773 + - type: nauc_map_at_1000_max + value: 31.0185 + - type: nauc_map_at_1000_std + value: 17.6212 + - type: nauc_map_at_1000_diff1 + value: 20.846700000000002 + - type: nauc_recall_at_1_max + value: 20.247200000000003 + - type: nauc_recall_at_1_std + value: 8.8046 + - type: nauc_recall_at_1_diff1 + value: 27.227600000000002 + - type: nauc_recall_at_3_max + value: 25.074600000000004 + - type: nauc_recall_at_3_std + value: 14.0657 + - type: nauc_recall_at_3_diff1 + value: 14.7258 + - type: nauc_recall_at_5_max + value: 29.442899999999998 + - type: nauc_recall_at_5_std + value: 16.2404 + - type: nauc_recall_at_5_diff1 + value: 15.4134 + - type: nauc_recall_at_10_max + value: 33.5052 + - type: nauc_recall_at_10_std + value: 19.417 + - type: nauc_recall_at_10_diff1 + value: 7.933700000000001 + - type: nauc_recall_at_20_max + value: 40.2402 + - type: nauc_recall_at_20_std + value: 22.7218 + - type: nauc_recall_at_20_diff1 + value: 11.777600000000001 + - type: nauc_recall_at_100_max + value: 44.4613 + - type: nauc_recall_at_100_std + value: 52.5751 + - type: nauc_recall_at_100_diff1 + value: 5.1827 + - type: nauc_recall_at_1000_max + value: 80.4059 + - type: nauc_recall_at_1000_std + value: 82.2582 + - type: nauc_recall_at_1000_diff1 + value: 37.9332 + - type: nauc_precision_at_1_max + value: 29.1135 + - type: nauc_precision_at_1_std + value: 13.9561 + - type: nauc_precision_at_1_diff1 + value: 28.410400000000003 + - type: nauc_precision_at_3_max + value: 32.4031 + - type: nauc_precision_at_3_std + value: 21.222099999999998 + - type: nauc_precision_at_3_diff1 + value: 9.2426 + - type: nauc_precision_at_5_max + value: 31.372600000000002 + - type: nauc_precision_at_5_std + value: 22.4259 + - type: nauc_precision_at_5_diff1 + value: 7.199 + - type: nauc_precision_at_10_max + value: 29.5298 + - type: nauc_precision_at_10_std + value: 22.183 + - type: nauc_precision_at_10_diff1 + value: -1.2202 + - type: nauc_precision_at_20_max + value: 28.1874 + - type: nauc_precision_at_20_std + value: 21.7393 + - type: nauc_precision_at_20_diff1 + value: 0.2774 + - type: nauc_precision_at_100_max + value: 18.2122 + - type: nauc_precision_at_100_std + value: 21.566 + - type: nauc_precision_at_100_diff1 + value: -5.8792 + - type: nauc_precision_at_1000_max + value: 11.3258 + - type: nauc_precision_at_1000_std + value: 12.261700000000001 + - type: nauc_precision_at_1000_diff1 + value: -5.8514 + - type: nauc_mrr_at_1_max + value: 29.1135 + - type: nauc_mrr_at_1_std + value: 13.9561 + - type: nauc_mrr_at_1_diff1 + value: 28.410400000000003 + - type: nauc_mrr_at_3_max + value: 30.904999999999998 + - type: nauc_mrr_at_3_std + value: 16.5695 + - type: nauc_mrr_at_3_diff1 + value: 22.555 + - type: nauc_mrr_at_5_max + value: 32.408 + - type: nauc_mrr_at_5_std + value: 17.7334 + - type: nauc_mrr_at_5_diff1 + value: 22.912399999999998 + - type: nauc_mrr_at_10_max + value: 33.069500000000005 + - type: nauc_mrr_at_10_std + value: 17.8731 + - type: nauc_mrr_at_10_diff1 + value: 22.270300000000002 + - type: nauc_mrr_at_20_max + value: 33.062000000000005 + - type: nauc_mrr_at_20_std + value: 17.8293 + - type: nauc_mrr_at_20_diff1 + value: 22.5118 + - type: nauc_mrr_at_100_max + value: 32.9394 + - type: nauc_mrr_at_100_std + value: 17.7815 + - type: nauc_mrr_at_100_diff1 + value: 22.676199999999998 + - type: nauc_mrr_at_1000_max + value: 32.9188 + - type: nauc_mrr_at_1000_std + value: 17.7435 + - type: nauc_mrr_at_1000_diff1 + value: 22.6855 + - type: main_score + value: 49.891000000000005 + task: + type: Retrieval + - dataset: + config: hi + name: MTEB MIRACLRetrieval (hi) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 36.857 + - type: ndcg_at_3 + value: 39.469 + - type: ndcg_at_5 + value: 41.839999999999996 + - type: ndcg_at_10 + value: 46.141 + - type: ndcg_at_20 + value: 49.384 + - type: ndcg_at_100 + value: 52.565 + - type: ndcg_at_1000 + value: 54.318999999999996 + - type: map_at_1 + value: 20.185 + - type: map_at_3 + value: 30.9 + - type: map_at_5 + value: 34.311 + - type: map_at_10 + value: 37.074 + - type: map_at_20 + value: 38.493 + - type: map_at_100 + value: 39.174 + - type: map_at_1000 + value: 39.269 + - type: recall_at_1 + value: 20.185 + - type: recall_at_3 + value: 38.993 + - type: recall_at_5 + value: 47.881 + - type: recall_at_10 + value: 59.474000000000004 + - type: recall_at_20 + value: 69.437 + - type: recall_at_100 + value: 83.38499999999999 + - type: recall_at_1000 + value: 94.813 + - type: precision_at_1 + value: 36.857 + - type: precision_at_3 + value: 26.19 + - type: precision_at_5 + value: 19.829 + - type: precision_at_10 + value: 12.543000000000001 + - type: precision_at_20 + value: 7.542999999999999 + - type: precision_at_100 + value: 1.8030000000000002 + - type: precision_at_1000 + value: 0.20500000000000002 + - type: mrr_at_1 + value: 36.857099999999996 + - type: mrr_at_3 + value: 46.5238 + - type: mrr_at_5 + value: 47.9952 + - type: mrr_at_10 + value: 49.331399999999995 + - type: mrr_at_20 + value: 49.8255 + - type: mrr_at_100 + value: 50.0575 + - type: mrr_at_1000 + value: 50.097 + - type: nauc_ndcg_at_1_max + value: 42.226200000000006 + - type: nauc_ndcg_at_1_std + value: 4.0359 + - type: nauc_ndcg_at_1_diff1 + value: 41.728500000000004 + - type: nauc_ndcg_at_3_max + value: 37.5731 + - type: nauc_ndcg_at_3_std + value: 7.4824 + - type: nauc_ndcg_at_3_diff1 + value: 25.607499999999998 + - type: nauc_ndcg_at_5_max + value: 36.1243 + - type: nauc_ndcg_at_5_std + value: 6.7822 + - type: nauc_ndcg_at_5_diff1 + value: 26.4955 + - type: nauc_ndcg_at_10_max + value: 38.8673 + - type: nauc_ndcg_at_10_std + value: 9.925699999999999 + - type: nauc_ndcg_at_10_diff1 + value: 25.262400000000003 + - type: nauc_ndcg_at_20_max + value: 41.564099999999996 + - type: nauc_ndcg_at_20_std + value: 12.4619 + - type: nauc_ndcg_at_20_diff1 + value: 26.902900000000002 + - type: nauc_ndcg_at_100_max + value: 42.2534 + - type: nauc_ndcg_at_100_std + value: 12.1461 + - type: nauc_ndcg_at_100_diff1 + value: 27.721600000000002 + - type: nauc_ndcg_at_1000_max + value: 42.3689 + - type: nauc_ndcg_at_1000_std + value: 11.9947 + - type: nauc_ndcg_at_1000_diff1 + value: 28.6224 + - type: nauc_map_at_1_max + value: 23.4774 + - type: nauc_map_at_1_std + value: -1.6596 + - type: nauc_map_at_1_diff1 + value: 32.9091 + - type: nauc_map_at_3_max + value: 29.2888 + - type: nauc_map_at_3_std + value: 2.8310999999999997 + - type: nauc_map_at_3_diff1 + value: 25.7556 + - type: nauc_map_at_5_max + value: 32.013200000000005 + - type: nauc_map_at_5_std + value: 3.8372 + - type: nauc_map_at_5_diff1 + value: 26.3662 + - type: nauc_map_at_10_max + value: 34.6644 + - type: nauc_map_at_10_std + value: 5.9211 + - type: nauc_map_at_10_diff1 + value: 25.737700000000004 + - type: nauc_map_at_20_max + value: 36.5315 + - type: nauc_map_at_20_std + value: 7.657500000000001 + - type: nauc_map_at_20_diff1 + value: 26.2519 + - type: nauc_map_at_100_max + value: 36.7956 + - type: nauc_map_at_100_std + value: 7.6282000000000005 + - type: nauc_map_at_100_diff1 + value: 26.5173 + - type: nauc_map_at_1000_max + value: 36.822500000000005 + - type: nauc_map_at_1000_std + value: 7.641100000000001 + - type: nauc_map_at_1000_diff1 + value: 26.5875 + - type: nauc_recall_at_1_max + value: 23.4774 + - type: nauc_recall_at_1_std + value: -1.6596 + - type: nauc_recall_at_1_diff1 + value: 32.9091 + - type: nauc_recall_at_3_max + value: 23.9443 + - type: nauc_recall_at_3_std + value: 7.0466 + - type: nauc_recall_at_3_diff1 + value: 15.045 + - type: nauc_recall_at_5_max + value: 27.515 + - type: nauc_recall_at_5_std + value: 7.8471 + - type: nauc_recall_at_5_diff1 + value: 16.0936 + - type: nauc_recall_at_10_max + value: 32.9675 + - type: nauc_recall_at_10_std + value: 15.6248 + - type: nauc_recall_at_10_diff1 + value: 11.8783 + - type: nauc_recall_at_20_max + value: 40.6864 + - type: nauc_recall_at_20_std + value: 23.9995 + - type: nauc_recall_at_20_diff1 + value: 16.9561 + - type: nauc_recall_at_100_max + value: 47.5027 + - type: nauc_recall_at_100_std + value: 30.6021 + - type: nauc_recall_at_100_diff1 + value: 17.3955 + - type: nauc_recall_at_1000_max + value: 66.6978 + - type: nauc_recall_at_1000_std + value: 62.0413 + - type: nauc_recall_at_1000_diff1 + value: 27.5068 + - type: nauc_precision_at_1_max + value: 42.226200000000006 + - type: nauc_precision_at_1_std + value: 4.0359 + - type: nauc_precision_at_1_diff1 + value: 41.728500000000004 + - type: nauc_precision_at_3_max + value: 44.7816 + - type: nauc_precision_at_3_std + value: 15.473300000000002 + - type: nauc_precision_at_3_diff1 + value: 17.0949 + - type: nauc_precision_at_5_max + value: 44.6483 + - type: nauc_precision_at_5_std + value: 14.8981 + - type: nauc_precision_at_5_diff1 + value: 17.1841 + - type: nauc_precision_at_10_max + value: 45.796 + - type: nauc_precision_at_10_std + value: 21.046300000000002 + - type: nauc_precision_at_10_diff1 + value: 10.9757 + - type: nauc_precision_at_20_max + value: 45.0264 + - type: nauc_precision_at_20_std + value: 24.8162 + - type: nauc_precision_at_20_diff1 + value: 10.624699999999999 + - type: nauc_precision_at_100_max + value: 39.8456 + - type: nauc_precision_at_100_std + value: 21.0487 + - type: nauc_precision_at_100_diff1 + value: 8.372 + - type: nauc_precision_at_1000_max + value: 34.7517 + - type: nauc_precision_at_1000_std + value: 18.3825 + - type: nauc_precision_at_1000_diff1 + value: 7.969900000000001 + - type: nauc_mrr_at_1_max + value: 42.226200000000006 + - type: nauc_mrr_at_1_std + value: 4.0359 + - type: nauc_mrr_at_1_diff1 + value: 41.728500000000004 + - type: nauc_mrr_at_3_max + value: 42.1134 + - type: nauc_mrr_at_3_std + value: 7.674799999999999 + - type: nauc_mrr_at_3_diff1 + value: 34.1447 + - type: nauc_mrr_at_5_max + value: 42.668800000000005 + - type: nauc_mrr_at_5_std + value: 7.3921 + - type: nauc_mrr_at_5_diff1 + value: 34.6011 + - type: nauc_mrr_at_10_max + value: 43.473099999999995 + - type: nauc_mrr_at_10_std + value: 8.0841 + - type: nauc_mrr_at_10_diff1 + value: 34.679500000000004 + - type: nauc_mrr_at_20_max + value: 43.3626 + - type: nauc_mrr_at_20_std + value: 7.7897 + - type: nauc_mrr_at_20_diff1 + value: 35.0828 + - type: nauc_mrr_at_100_max + value: 43.287 + - type: nauc_mrr_at_100_std + value: 7.7234 + - type: nauc_mrr_at_100_diff1 + value: 35.169200000000004 + - type: nauc_mrr_at_1000_max + value: 43.2954 + - type: nauc_mrr_at_1000_std + value: 7.7224 + - type: nauc_mrr_at_1000_diff1 + value: 35.1808 + - type: main_score + value: 46.141 + task: + type: Retrieval + - dataset: + config: id + name: MTEB MIRACLRetrieval (id) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 46.354 + - type: ndcg_at_3 + value: 42.538 + - type: ndcg_at_5 + value: 43.717 + - type: ndcg_at_10 + value: 47.229 + - type: ndcg_at_20 + value: 50.605999999999995 + - type: ndcg_at_100 + value: 55.25 + - type: ndcg_at_1000 + value: 57.647999999999996 + - type: map_at_1 + value: 20.787 + - type: map_at_3 + value: 30.721999999999998 + - type: map_at_5 + value: 34.096 + - type: map_at_10 + value: 36.994 + - type: map_at_20 + value: 38.622 + - type: map_at_100 + value: 39.872 + - type: map_at_1000 + value: 40.056000000000004 + - type: recall_at_1 + value: 20.787 + - type: recall_at_3 + value: 36.229 + - type: recall_at_5 + value: 44.437 + - type: recall_at_10 + value: 54.771 + - type: recall_at_20 + value: 63.842 + - type: recall_at_100 + value: 80.689 + - type: recall_at_1000 + value: 94.03200000000001 + - type: precision_at_1 + value: 46.354 + - type: precision_at_3 + value: 30.625000000000004 + - type: precision_at_5 + value: 23.708000000000002 + - type: precision_at_10 + value: 15.719 + - type: precision_at_20 + value: 9.589 + - type: precision_at_100 + value: 2.5700000000000003 + - type: precision_at_1000 + value: 0.302 + - type: mrr_at_1 + value: 46.3542 + - type: mrr_at_3 + value: 54.6875 + - type: mrr_at_5 + value: 56.5521 + - type: mrr_at_10 + value: 57.6894 + - type: mrr_at_20 + value: 58.05630000000001 + - type: mrr_at_100 + value: 58.217 + - type: mrr_at_1000 + value: 58.2387 + - type: nauc_ndcg_at_1_max + value: 27.987000000000002 + - type: nauc_ndcg_at_1_std + value: 7.784000000000001 + - type: nauc_ndcg_at_1_diff1 + value: 29.116799999999998 + - type: nauc_ndcg_at_3_max + value: 25.316899999999997 + - type: nauc_ndcg_at_3_std + value: 3.3255 + - type: nauc_ndcg_at_3_diff1 + value: 25.4685 + - type: nauc_ndcg_at_5_max + value: 26.1614 + - type: nauc_ndcg_at_5_std + value: 0.8946000000000001 + - type: nauc_ndcg_at_5_diff1 + value: 25.269799999999996 + - type: nauc_ndcg_at_10_max + value: 26.898 + - type: nauc_ndcg_at_10_std + value: 0.505 + - type: nauc_ndcg_at_10_diff1 + value: 25.0664 + - type: nauc_ndcg_at_20_max + value: 28.384900000000002 + - type: nauc_ndcg_at_20_std + value: 3.0328 + - type: nauc_ndcg_at_20_diff1 + value: 25.011 + - type: nauc_ndcg_at_100_max + value: 29.4682 + - type: nauc_ndcg_at_100_std + value: 8.5929 + - type: nauc_ndcg_at_100_diff1 + value: 23.0951 + - type: nauc_ndcg_at_1000_max + value: 29.384900000000002 + - type: nauc_ndcg_at_1000_std + value: 8.7787 + - type: nauc_ndcg_at_1000_diff1 + value: 23.454900000000002 + - type: nauc_map_at_1_max + value: 17.6022 + - type: nauc_map_at_1_std + value: -3.9352 + - type: nauc_map_at_1_diff1 + value: 31.478 + - type: nauc_map_at_3_max + value: 22.4116 + - type: nauc_map_at_3_std + value: -3.0375 + - type: nauc_map_at_3_diff1 + value: 28.6608 + - type: nauc_map_at_5_max + value: 23.4486 + - type: nauc_map_at_5_std + value: -3.7261 + - type: nauc_map_at_5_diff1 + value: 27.2458 + - type: nauc_map_at_10_max + value: 24.4413 + - type: nauc_map_at_10_std + value: -2.4634 + - type: nauc_map_at_10_diff1 + value: 26.3372 + - type: nauc_map_at_20_max + value: 25.1924 + - type: nauc_map_at_20_std + value: -1.0928 + - type: nauc_map_at_20_diff1 + value: 26.028299999999998 + - type: nauc_map_at_100_max + value: 25.7081 + - type: nauc_map_at_100_std + value: 0.6245999999999999 + - type: nauc_map_at_100_diff1 + value: 25.599 + - type: nauc_map_at_1000_max + value: 25.714100000000002 + - type: nauc_map_at_1000_std + value: 0.7106 + - type: nauc_map_at_1000_diff1 + value: 25.609700000000004 + - type: nauc_recall_at_1_max + value: 17.6022 + - type: nauc_recall_at_1_std + value: -3.9352 + - type: nauc_recall_at_1_diff1 + value: 31.478 + - type: nauc_recall_at_3_max + value: 20.314799999999998 + - type: nauc_recall_at_3_std + value: -4.1603 + - type: nauc_recall_at_3_diff1 + value: 26.1438 + - type: nauc_recall_at_5_max + value: 22.866500000000002 + - type: nauc_recall_at_5_std + value: -4.755 + - type: nauc_recall_at_5_diff1 + value: 22.1412 + - type: nauc_recall_at_10_max + value: 22.900000000000002 + - type: nauc_recall_at_10_std + value: -3.9179 + - type: nauc_recall_at_10_diff1 + value: 19.3005 + - type: nauc_recall_at_20_max + value: 26.3519 + - type: nauc_recall_at_20_std + value: 1.1686 + - type: nauc_recall_at_20_diff1 + value: 18.94 + - type: nauc_recall_at_100_max + value: 30.2413 + - type: nauc_recall_at_100_std + value: 24.4636 + - type: nauc_recall_at_100_diff1 + value: 6.5627 + - type: nauc_recall_at_1000_max + value: 43.778 + - type: nauc_recall_at_1000_std + value: 48.835699999999996 + - type: nauc_recall_at_1000_diff1 + value: -1.5112 + - type: nauc_precision_at_1_max + value: 27.987000000000002 + - type: nauc_precision_at_1_std + value: 7.784000000000001 + - type: nauc_precision_at_1_diff1 + value: 29.116799999999998 + - type: nauc_precision_at_3_max + value: 24.6393 + - type: nauc_precision_at_3_std + value: 7.932599999999999 + - type: nauc_precision_at_3_diff1 + value: 11.9215 + - type: nauc_precision_at_5_max + value: 23.0426 + - type: nauc_precision_at_5_std + value: 8.9273 + - type: nauc_precision_at_5_diff1 + value: 5.0737 + - type: nauc_precision_at_10_max + value: 18.0093 + - type: nauc_precision_at_10_std + value: 13.093 + - type: nauc_precision_at_10_diff1 + value: -1.5028 + - type: nauc_precision_at_20_max + value: 16.1061 + - type: nauc_precision_at_20_std + value: 18.3582 + - type: nauc_precision_at_20_diff1 + value: -4.3066 + - type: nauc_precision_at_100_max + value: 10.9945 + - type: nauc_precision_at_100_std + value: 28.2804 + - type: nauc_precision_at_100_diff1 + value: -11.6381 + - type: nauc_precision_at_1000_max + value: 4.9859 + - type: nauc_precision_at_1000_std + value: 26.3117 + - type: nauc_precision_at_1000_diff1 + value: -13.819300000000002 + - type: nauc_mrr_at_1_max + value: 27.987000000000002 + - type: nauc_mrr_at_1_std + value: 7.784000000000001 + - type: nauc_mrr_at_1_diff1 + value: 29.116799999999998 + - type: nauc_mrr_at_3_max + value: 28.635899999999996 + - type: nauc_mrr_at_3_std + value: 8.309700000000001 + - type: nauc_mrr_at_3_diff1 + value: 27.976499999999998 + - type: nauc_mrr_at_5_max + value: 29.8296 + - type: nauc_mrr_at_5_std + value: 9.4775 + - type: nauc_mrr_at_5_diff1 + value: 26.685799999999997 + - type: nauc_mrr_at_10_max + value: 29.4522 + - type: nauc_mrr_at_10_std + value: 9.1613 + - type: nauc_mrr_at_10_diff1 + value: 26.933600000000002 + - type: nauc_mrr_at_20_max + value: 29.5446 + - type: nauc_mrr_at_20_std + value: 9.3451 + - type: nauc_mrr_at_20_diff1 + value: 27.074900000000003 + - type: nauc_mrr_at_100_max + value: 29.4977 + - type: nauc_mrr_at_100_std + value: 9.4252 + - type: nauc_mrr_at_100_diff1 + value: 27.0534 + - type: nauc_mrr_at_1000_max + value: 29.499599999999997 + - type: nauc_mrr_at_1000_std + value: 9.4193 + - type: nauc_mrr_at_1000_diff1 + value: 27.054000000000002 + - type: main_score + value: 47.229 + task: + type: Retrieval + - dataset: + config: ja + name: MTEB MIRACLRetrieval (ja) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 56.279 + - type: ndcg_at_3 + value: 56.226 + - type: ndcg_at_5 + value: 58.660000000000004 + - type: ndcg_at_10 + value: 62.81 + - type: ndcg_at_20 + value: 65.21000000000001 + - type: ndcg_at_100 + value: 67.757 + - type: ndcg_at_1000 + value: 68.667 + - type: map_at_1 + value: 36.647999999999996 + - type: map_at_3 + value: 48.154 + - type: map_at_5 + value: 51.336999999999996 + - type: map_at_10 + value: 53.998000000000005 + - type: map_at_20 + value: 55.074 + - type: map_at_100 + value: 55.701 + - type: map_at_1000 + value: 55.767 + - type: recall_at_1 + value: 36.647999999999996 + - type: recall_at_3 + value: 55.845 + - type: recall_at_5 + value: 63.854 + - type: recall_at_10 + value: 74.96000000000001 + - type: recall_at_20 + value: 82.326 + - type: recall_at_100 + value: 92.461 + - type: recall_at_1000 + value: 97.827 + - type: precision_at_1 + value: 56.279 + - type: precision_at_3 + value: 31.86 + - type: precision_at_5 + value: 22.884 + - type: precision_at_10 + value: 14.058000000000002 + - type: precision_at_20 + value: 7.965 + - type: precision_at_100 + value: 1.883 + - type: precision_at_1000 + value: 0.203 + - type: mrr_at_1 + value: 56.27910000000001 + - type: mrr_at_3 + value: 64.7868 + - type: mrr_at_5 + value: 65.9496 + - type: mrr_at_10 + value: 67.0763 + - type: mrr_at_20 + value: 67.3531 + - type: mrr_at_100 + value: 67.48920000000001 + - type: mrr_at_1000 + value: 67.5016 + - type: nauc_ndcg_at_1_max + value: 34.801300000000005 + - type: nauc_ndcg_at_1_std + value: 3.6539 + - type: nauc_ndcg_at_1_diff1 + value: 42.9912 + - type: nauc_ndcg_at_3_max + value: 27.3758 + - type: nauc_ndcg_at_3_std + value: -5.6399 + - type: nauc_ndcg_at_3_diff1 + value: 35.0235 + - type: nauc_ndcg_at_5_max + value: 26.5087 + - type: nauc_ndcg_at_5_std + value: -7.2121 + - type: nauc_ndcg_at_5_diff1 + value: 34.3684 + - type: nauc_ndcg_at_10_max + value: 27.756199999999996 + - type: nauc_ndcg_at_10_std + value: -6.9499 + - type: nauc_ndcg_at_10_diff1 + value: 34.9472 + - type: nauc_ndcg_at_20_max + value: 30.6925 + - type: nauc_ndcg_at_20_std + value: -3.7859 + - type: nauc_ndcg_at_20_diff1 + value: 35.833 + - type: nauc_ndcg_at_100_max + value: 31.6641 + - type: nauc_ndcg_at_100_std + value: -1.1897 + - type: nauc_ndcg_at_100_diff1 + value: 36.218 + - type: nauc_ndcg_at_1000_max + value: 31.5623 + - type: nauc_ndcg_at_1000_std + value: -1.2468 + - type: nauc_ndcg_at_1000_diff1 + value: 36.4007 + - type: nauc_map_at_1_max + value: 13.1087 + - type: nauc_map_at_1_std + value: -13.6324 + - type: nauc_map_at_1_diff1 + value: 36.5411 + - type: nauc_map_at_3_max + value: 19.108900000000002 + - type: nauc_map_at_3_std + value: -12.8558 + - type: nauc_map_at_3_diff1 + value: 33.797 + - type: nauc_map_at_5_max + value: 20.935100000000002 + - type: nauc_map_at_5_std + value: -11.6525 + - type: nauc_map_at_5_diff1 + value: 33.392500000000005 + - type: nauc_map_at_10_max + value: 22.9758 + - type: nauc_map_at_10_std + value: -10.3728 + - type: nauc_map_at_10_diff1 + value: 33.8681 + - type: nauc_map_at_20_max + value: 24.357100000000003 + - type: nauc_map_at_20_std + value: -8.9932 + - type: nauc_map_at_20_diff1 + value: 34.2437 + - type: nauc_map_at_100_max + value: 24.622700000000002 + - type: nauc_map_at_100_std + value: -8.3079 + - type: nauc_map_at_100_diff1 + value: 34.3227 + - type: nauc_map_at_1000_max + value: 24.6436 + - type: nauc_map_at_1000_std + value: -8.280999999999999 + - type: nauc_map_at_1000_diff1 + value: 34.3499 + - type: nauc_recall_at_1_max + value: 13.1087 + - type: nauc_recall_at_1_std + value: -13.6324 + - type: nauc_recall_at_1_diff1 + value: 36.5411 + - type: nauc_recall_at_3_max + value: 17.369899999999998 + - type: nauc_recall_at_3_std + value: -14.6564 + - type: nauc_recall_at_3_diff1 + value: 29.4825 + - type: nauc_recall_at_5_max + value: 18.2446 + - type: nauc_recall_at_5_std + value: -13.422400000000001 + - type: nauc_recall_at_5_diff1 + value: 26.5515 + - type: nauc_recall_at_10_max + value: 18.6431 + - type: nauc_recall_at_10_std + value: -13.3386 + - type: nauc_recall_at_10_diff1 + value: 25.001299999999997 + - type: nauc_recall_at_20_max + value: 28.248099999999997 + - type: nauc_recall_at_20_std + value: -2.9409 + - type: nauc_recall_at_20_diff1 + value: 26.283800000000003 + - type: nauc_recall_at_100_max + value: 38.6213 + - type: nauc_recall_at_100_std + value: 20.5175 + - type: nauc_recall_at_100_diff1 + value: 23.8743 + - type: nauc_recall_at_1000_max + value: 54.1945 + - type: nauc_recall_at_1000_std + value: 48.3776 + - type: nauc_recall_at_1000_diff1 + value: 21.786 + - type: nauc_precision_at_1_max + value: 34.801300000000005 + - type: nauc_precision_at_1_std + value: 3.6539 + - type: nauc_precision_at_1_diff1 + value: 42.9912 + - type: nauc_precision_at_3_max + value: 36.7085 + - type: nauc_precision_at_3_std + value: 13.653799999999999 + - type: nauc_precision_at_3_diff1 + value: 16.8438 + - type: nauc_precision_at_5_max + value: 33.541199999999996 + - type: nauc_precision_at_5_std + value: 17.418400000000002 + - type: nauc_precision_at_5_diff1 + value: 8.5281 + - type: nauc_precision_at_10_max + value: 32.448100000000004 + - type: nauc_precision_at_10_std + value: 22.8249 + - type: nauc_precision_at_10_diff1 + value: 2.5392 + - type: nauc_precision_at_20_max + value: 32.423 + - type: nauc_precision_at_20_std + value: 29.353800000000003 + - type: nauc_precision_at_20_diff1 + value: 0.1455 + - type: nauc_precision_at_100_max + value: 25.0045 + - type: nauc_precision_at_100_std + value: 34.6492 + - type: nauc_precision_at_100_diff1 + value: -5.5314000000000005 + - type: nauc_precision_at_1000_max + value: 21.319499999999998 + - type: nauc_precision_at_1000_std + value: 33.3312 + - type: nauc_precision_at_1000_diff1 + value: -7.0243 + - type: nauc_mrr_at_1_max + value: 34.801300000000005 + - type: nauc_mrr_at_1_std + value: 3.6539 + - type: nauc_mrr_at_1_diff1 + value: 42.9912 + - type: nauc_mrr_at_3_max + value: 39.8179 + - type: nauc_mrr_at_3_std + value: 4.4769000000000005 + - type: nauc_mrr_at_3_diff1 + value: 42.4358 + - type: nauc_mrr_at_5_max + value: 39.6822 + - type: nauc_mrr_at_5_std + value: 4.7865 + - type: nauc_mrr_at_5_diff1 + value: 41.9923 + - type: nauc_mrr_at_10_max + value: 39.2963 + - type: nauc_mrr_at_10_std + value: 4.8511 + - type: nauc_mrr_at_10_diff1 + value: 41.994 + - type: nauc_mrr_at_20_max + value: 39.395799999999994 + - type: nauc_mrr_at_20_std + value: 4.9907 + - type: nauc_mrr_at_20_diff1 + value: 42.1806 + - type: nauc_mrr_at_100_max + value: 39.3251 + - type: nauc_mrr_at_100_std + value: 4.948 + - type: nauc_mrr_at_100_diff1 + value: 42.1769 + - type: nauc_mrr_at_1000_max + value: 39.3153 + - type: nauc_mrr_at_1000_std + value: 4.9384999999999994 + - type: nauc_mrr_at_1000_diff1 + value: 42.1768 + - type: main_score + value: 62.81 + task: + type: Retrieval + - dataset: + config: ko + name: MTEB MIRACLRetrieval (ko) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 52.581999999999994 + - type: ndcg_at_3 + value: 53.73 + - type: ndcg_at_5 + value: 55.886 + - type: ndcg_at_10 + value: 59.216 + - type: ndcg_at_20 + value: 62.427 + - type: ndcg_at_100 + value: 65.093 + - type: ndcg_at_1000 + value: 66.204 + - type: map_at_1 + value: 30.520999999999997 + - type: map_at_3 + value: 42.601 + - type: map_at_5 + value: 46.516000000000005 + - type: map_at_10 + value: 49.61 + - type: map_at_20 + value: 51.359 + - type: map_at_100 + value: 52.171 + - type: map_at_1000 + value: 52.249 + - type: recall_at_1 + value: 30.520999999999997 + - type: recall_at_3 + value: 51.5 + - type: recall_at_5 + value: 60.709999999999994 + - type: recall_at_10 + value: 71.15899999999999 + - type: recall_at_20 + value: 80.209 + - type: recall_at_100 + value: 90.203 + - type: recall_at_1000 + value: 96.714 + - type: precision_at_1 + value: 52.581999999999994 + - type: precision_at_3 + value: 33.019999999999996 + - type: precision_at_5 + value: 25.446 + - type: precision_at_10 + value: 16.244 + - type: precision_at_20 + value: 9.695 + - type: precision_at_100 + value: 2.286 + - type: precision_at_1000 + value: 0.248 + - type: mrr_at_1 + value: 52.5822 + - type: mrr_at_3 + value: 61.9718 + - type: mrr_at_5 + value: 63.450700000000005 + - type: mrr_at_10 + value: 64.50479999999999 + - type: mrr_at_20 + value: 64.7745 + - type: mrr_at_100 + value: 64.86840000000001 + - type: mrr_at_1000 + value: 64.8792 + - type: nauc_ndcg_at_1_max + value: 57.2789 + - type: nauc_ndcg_at_1_std + value: 34.9863 + - type: nauc_ndcg_at_1_diff1 + value: 44.0111 + - type: nauc_ndcg_at_3_max + value: 34.18 + - type: nauc_ndcg_at_3_std + value: 11.1503 + - type: nauc_ndcg_at_3_diff1 + value: 40.339999999999996 + - type: nauc_ndcg_at_5_max + value: 34.4364 + - type: nauc_ndcg_at_5_std + value: 8.7133 + - type: nauc_ndcg_at_5_diff1 + value: 43.3464 + - type: nauc_ndcg_at_10_max + value: 35.990899999999996 + - type: nauc_ndcg_at_10_std + value: 10.886700000000001 + - type: nauc_ndcg_at_10_diff1 + value: 43.3519 + - type: nauc_ndcg_at_20_max + value: 40.259499999999996 + - type: nauc_ndcg_at_20_std + value: 16.305600000000002 + - type: nauc_ndcg_at_20_diff1 + value: 43.526900000000005 + - type: nauc_ndcg_at_100_max + value: 44.4663 + - type: nauc_ndcg_at_100_std + value: 21.5157 + - type: nauc_ndcg_at_100_diff1 + value: 43.269999999999996 + - type: nauc_ndcg_at_1000_max + value: 44.5037 + - type: nauc_ndcg_at_1000_std + value: 21.6384 + - type: nauc_ndcg_at_1000_diff1 + value: 43.5169 + - type: nauc_map_at_1_max + value: 9.6775 + - type: nauc_map_at_1_std + value: -7.5287999999999995 + - type: nauc_map_at_1_diff1 + value: 56.714200000000005 + - type: nauc_map_at_3_max + value: 14.175199999999998 + - type: nauc_map_at_3_std + value: -9.251800000000001 + - type: nauc_map_at_3_diff1 + value: 47.239 + - type: nauc_map_at_5_max + value: 20.4059 + - type: nauc_map_at_5_std + value: -3.9799 + - type: nauc_map_at_5_diff1 + value: 46.5588 + - type: nauc_map_at_10_max + value: 26.7796 + - type: nauc_map_at_10_std + value: 2.3718 + - type: nauc_map_at_10_diff1 + value: 45.5976 + - type: nauc_map_at_20_max + value: 30.291400000000003 + - type: nauc_map_at_20_std + value: 6.3573 + - type: nauc_map_at_20_diff1 + value: 45.5914 + - type: nauc_map_at_100_max + value: 32.0062 + - type: nauc_map_at_100_std + value: 8.2968 + - type: nauc_map_at_100_diff1 + value: 45.6306 + - type: nauc_map_at_1000_max + value: 32.0482 + - type: nauc_map_at_1000_std + value: 8.3688 + - type: nauc_map_at_1000_diff1 + value: 45.6447 + - type: nauc_recall_at_1_max + value: 9.6775 + - type: nauc_recall_at_1_std + value: -7.5287999999999995 + - type: nauc_recall_at_1_diff1 + value: 56.714200000000005 + - type: nauc_recall_at_3_max + value: 4.7592 + - type: nauc_recall_at_3_std + value: -17.7268 + - type: nauc_recall_at_3_diff1 + value: 36.593599999999995 + - type: nauc_recall_at_5_max + value: 11.0166 + - type: nauc_recall_at_5_std + value: -14.832799999999999 + - type: nauc_recall_at_5_diff1 + value: 36.6471 + - type: nauc_recall_at_10_max + value: 20.272299999999998 + - type: nauc_recall_at_10_std + value: -3.9745000000000004 + - type: nauc_recall_at_10_diff1 + value: 34.875699999999995 + - type: nauc_recall_at_20_max + value: 27.0707 + - type: nauc_recall_at_20_std + value: 5.8709 + - type: nauc_recall_at_20_diff1 + value: 34.921600000000005 + - type: nauc_recall_at_100_max + value: 48.045100000000005 + - type: nauc_recall_at_100_std + value: 32.3099 + - type: nauc_recall_at_100_diff1 + value: 30.127 + - type: nauc_recall_at_1000_max + value: 60.827299999999994 + - type: nauc_recall_at_1000_std + value: 49.6791 + - type: nauc_recall_at_1000_diff1 + value: 32.2816 + - type: nauc_precision_at_1_max + value: 57.2789 + - type: nauc_precision_at_1_std + value: 34.9863 + - type: nauc_precision_at_1_diff1 + value: 44.0111 + - type: nauc_precision_at_3_max + value: 55.550900000000006 + - type: nauc_precision_at_3_std + value: 39.1605 + - type: nauc_precision_at_3_diff1 + value: 2.1411 + - type: nauc_precision_at_5_max + value: 60.1216 + - type: nauc_precision_at_5_std + value: 49.1925 + - type: nauc_precision_at_5_diff1 + value: -4.2296 + - type: nauc_precision_at_10_max + value: 63.53339999999999 + - type: nauc_precision_at_10_std + value: 57.2366 + - type: nauc_precision_at_10_diff1 + value: -9.1914 + - type: nauc_precision_at_20_max + value: 63.2997 + - type: nauc_precision_at_20_std + value: 62.778 + - type: nauc_precision_at_20_diff1 + value: -11.4618 + - type: nauc_precision_at_100_max + value: 61.345000000000006 + - type: nauc_precision_at_100_std + value: 66.3033 + - type: nauc_precision_at_100_diff1 + value: -14.8779 + - type: nauc_precision_at_1000_max + value: 56.28300000000001 + - type: nauc_precision_at_1000_std + value: 62.91290000000001 + - type: nauc_precision_at_1000_diff1 + value: -16.6149 + - type: nauc_mrr_at_1_max + value: 57.2789 + - type: nauc_mrr_at_1_std + value: 34.9863 + - type: nauc_mrr_at_1_diff1 + value: 44.0111 + - type: nauc_mrr_at_3_max + value: 57.678200000000004 + - type: nauc_mrr_at_3_std + value: 33.5744 + - type: nauc_mrr_at_3_diff1 + value: 39.5643 + - type: nauc_mrr_at_5_max + value: 58.668600000000005 + - type: nauc_mrr_at_5_std + value: 33.5118 + - type: nauc_mrr_at_5_diff1 + value: 40.888200000000005 + - type: nauc_mrr_at_10_max + value: 58.4754 + - type: nauc_mrr_at_10_std + value: 33.7964 + - type: nauc_mrr_at_10_diff1 + value: 41.314 + - type: nauc_mrr_at_20_max + value: 58.434 + - type: nauc_mrr_at_20_std + value: 33.903 + - type: nauc_mrr_at_20_diff1 + value: 41.217999999999996 + - type: nauc_mrr_at_100_max + value: 58.4576 + - type: nauc_mrr_at_100_std + value: 33.9478 + - type: nauc_mrr_at_100_diff1 + value: 41.172599999999996 + - type: nauc_mrr_at_1000_max + value: 58.444399999999995 + - type: nauc_mrr_at_1000_std + value: 33.9292 + - type: nauc_mrr_at_1000_diff1 + value: 41.166199999999996 + - type: main_score + value: 59.216 + task: + type: Retrieval + - dataset: + config: ru + name: MTEB MIRACLRetrieval (ru) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 47.524 + - type: ndcg_at_3 + value: 46.812 + - type: ndcg_at_5 + value: 48.442 + - type: ndcg_at_10 + value: 52.349000000000004 + - type: ndcg_at_20 + value: 55.669000000000004 + - type: ndcg_at_100 + value: 59.724999999999994 + - type: ndcg_at_1000 + value: 61.312999999999995 + - type: map_at_1 + value: 24.337 + - type: map_at_3 + value: 35.765 + - type: map_at_5 + value: 39.153 + - type: map_at_10 + value: 42.225 + - type: map_at_20 + value: 43.782 + - type: map_at_100 + value: 44.887 + - type: map_at_1000 + value: 45.013 + - type: recall_at_1 + value: 24.337 + - type: recall_at_3 + value: 42.927 + - type: recall_at_5 + value: 51.258 + - type: recall_at_10 + value: 62.437 + - type: recall_at_20 + value: 71.411 + - type: recall_at_100 + value: 86.489 + - type: recall_at_1000 + value: 95.26599999999999 + - type: precision_at_1 + value: 47.524 + - type: precision_at_3 + value: 31.948999999999998 + - type: precision_at_5 + value: 24.121000000000002 + - type: precision_at_10 + value: 15.534999999999998 + - type: precision_at_20 + value: 9.408999999999999 + - type: precision_at_100 + value: 2.407 + - type: precision_at_1000 + value: 0.271 + - type: mrr_at_1 + value: 47.524 + - type: mrr_at_3 + value: 57.6012 + - type: mrr_at_5 + value: 59.130700000000004 + - type: mrr_at_10 + value: 60.1824 + - type: mrr_at_20 + value: 60.507200000000005 + - type: mrr_at_100 + value: 60.6675 + - type: mrr_at_1000 + value: 60.6789 + - type: nauc_ndcg_at_1_max + value: 32.3091 + - type: nauc_ndcg_at_1_std + value: 10.915700000000001 + - type: nauc_ndcg_at_1_diff1 + value: 35.0477 + - type: nauc_ndcg_at_3_max + value: 30.5579 + - type: nauc_ndcg_at_3_std + value: 9.9651 + - type: nauc_ndcg_at_3_diff1 + value: 28.537200000000002 + - type: nauc_ndcg_at_5_max + value: 30.7637 + - type: nauc_ndcg_at_5_std + value: 9.7618 + - type: nauc_ndcg_at_5_diff1 + value: 28.225699999999996 + - type: nauc_ndcg_at_10_max + value: 32.0146 + - type: nauc_ndcg_at_10_std + value: 9.681099999999999 + - type: nauc_ndcg_at_10_diff1 + value: 27.6866 + - type: nauc_ndcg_at_20_max + value: 34.7846 + - type: nauc_ndcg_at_20_std + value: 13.270599999999998 + - type: nauc_ndcg_at_20_diff1 + value: 27.8097 + - type: nauc_ndcg_at_100_max + value: 37.1031 + - type: nauc_ndcg_at_100_std + value: 16.512 + - type: nauc_ndcg_at_100_diff1 + value: 28.294200000000004 + - type: nauc_ndcg_at_1000_max + value: 36.5248 + - type: nauc_ndcg_at_1000_std + value: 16.1206 + - type: nauc_ndcg_at_1000_diff1 + value: 28.6308 + - type: nauc_map_at_1_max + value: 17.363300000000002 + - type: nauc_map_at_1_std + value: -3.3156 + - type: nauc_map_at_1_diff1 + value: 33.9402 + - type: nauc_map_at_3_max + value: 23.0235 + - type: nauc_map_at_3_std + value: 1.2713999999999999 + - type: nauc_map_at_3_diff1 + value: 28.946499999999997 + - type: nauc_map_at_5_max + value: 25.8014 + - type: nauc_map_at_5_std + value: 3.8541 + - type: nauc_map_at_5_diff1 + value: 28.526 + - type: nauc_map_at_10_max + value: 27.6617 + - type: nauc_map_at_10_std + value: 5.2938 + - type: nauc_map_at_10_diff1 + value: 28.122700000000002 + - type: nauc_map_at_20_max + value: 29.071399999999997 + - type: nauc_map_at_20_std + value: 7.005 + - type: nauc_map_at_20_diff1 + value: 28.075 + - type: nauc_map_at_100_max + value: 29.9533 + - type: nauc_map_at_100_std + value: 8.0838 + - type: nauc_map_at_100_diff1 + value: 28.2424 + - type: nauc_map_at_1000_max + value: 29.936200000000003 + - type: nauc_map_at_1000_std + value: 8.0967 + - type: nauc_map_at_1000_diff1 + value: 28.259 + - type: nauc_recall_at_1_max + value: 17.363300000000002 + - type: nauc_recall_at_1_std + value: -3.3156 + - type: nauc_recall_at_1_diff1 + value: 33.9402 + - type: nauc_recall_at_3_max + value: 20.7272 + - type: nauc_recall_at_3_std + value: 1.9171 + - type: nauc_recall_at_3_diff1 + value: 23.505300000000002 + - type: nauc_recall_at_5_max + value: 24.55 + - type: nauc_recall_at_5_std + value: 6.1491999999999996 + - type: nauc_recall_at_5_diff1 + value: 21.1769 + - type: nauc_recall_at_10_max + value: 26.6134 + - type: nauc_recall_at_10_std + value: 7.3684 + - type: nauc_recall_at_10_diff1 + value: 18.0016 + - type: nauc_recall_at_20_max + value: 33.744 + - type: nauc_recall_at_20_std + value: 17.2573 + - type: nauc_recall_at_20_diff1 + value: 17.3872 + - type: nauc_recall_at_100_max + value: 49.5745 + - type: nauc_recall_at_100_std + value: 39.4003 + - type: nauc_recall_at_100_diff1 + value: 16.1814 + - type: nauc_recall_at_1000_max + value: 62.5842 + - type: nauc_recall_at_1000_std + value: 64.7392 + - type: nauc_recall_at_1000_diff1 + value: 16.9464 + - type: nauc_precision_at_1_max + value: 32.3091 + - type: nauc_precision_at_1_std + value: 10.915700000000001 + - type: nauc_precision_at_1_diff1 + value: 35.0477 + - type: nauc_precision_at_3_max + value: 34.9888 + - type: nauc_precision_at_3_std + value: 22.009600000000002 + - type: nauc_precision_at_3_diff1 + value: 13.4801 + - type: nauc_precision_at_5_max + value: 34.1539 + - type: nauc_precision_at_5_std + value: 25.2388 + - type: nauc_precision_at_5_diff1 + value: 8.622 + - type: nauc_precision_at_10_max + value: 31.194 + - type: nauc_precision_at_10_std + value: 25.397100000000002 + - type: nauc_precision_at_10_diff1 + value: 3.4173 + - type: nauc_precision_at_20_max + value: 29.3116 + - type: nauc_precision_at_20_std + value: 28.8229 + - type: nauc_precision_at_20_diff1 + value: -0.4374 + - type: nauc_precision_at_100_max + value: 23.853099999999998 + - type: nauc_precision_at_100_std + value: 29.942800000000002 + - type: nauc_precision_at_100_diff1 + value: -3.9575 + - type: nauc_precision_at_1000_max + value: 16.5958 + - type: nauc_precision_at_1000_std + value: 25.208599999999997 + - type: nauc_precision_at_1000_diff1 + value: -6.1125 + - type: nauc_mrr_at_1_max + value: 32.3091 + - type: nauc_mrr_at_1_std + value: 10.915700000000001 + - type: nauc_mrr_at_1_diff1 + value: 35.0477 + - type: nauc_mrr_at_3_max + value: 36.9469 + - type: nauc_mrr_at_3_std + value: 15.4767 + - type: nauc_mrr_at_3_diff1 + value: 33.3922 + - type: nauc_mrr_at_5_max + value: 37.7043 + - type: nauc_mrr_at_5_std + value: 16.2089 + - type: nauc_mrr_at_5_diff1 + value: 33.3182 + - type: nauc_mrr_at_10_max + value: 37.5403 + - type: nauc_mrr_at_10_std + value: 16.229599999999998 + - type: nauc_mrr_at_10_diff1 + value: 33.2431 + - type: nauc_mrr_at_20_max + value: 37.4812 + - type: nauc_mrr_at_20_std + value: 16.278100000000002 + - type: nauc_mrr_at_20_diff1 + value: 33.3127 + - type: nauc_mrr_at_100_max + value: 37.43 + - type: nauc_mrr_at_100_std + value: 16.2077 + - type: nauc_mrr_at_100_diff1 + value: 33.3439 + - type: nauc_mrr_at_1000_max + value: 37.4133 + - type: nauc_mrr_at_1000_std + value: 16.1859 + - type: nauc_mrr_at_1000_diff1 + value: 33.353300000000004 + - type: main_score + value: 52.349000000000004 + task: + type: Retrieval + - dataset: + config: sw + name: MTEB MIRACLRetrieval (sw) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 51.66 + - type: ndcg_at_3 + value: 54.827999999999996 + - type: ndcg_at_5 + value: 57.382 + - type: ndcg_at_10 + value: 61.271 + - type: ndcg_at_20 + value: 63.64300000000001 + - type: ndcg_at_100 + value: 66.09899999999999 + - type: ndcg_at_1000 + value: 66.867 + - type: map_at_1 + value: 35.276999999999994 + - type: map_at_3 + value: 48.260999999999996 + - type: map_at_5 + value: 51.029 + - type: map_at_10 + value: 53.405 + - type: map_at_20 + value: 54.298 + - type: map_at_100 + value: 54.836 + - type: map_at_1000 + value: 54.887 + - type: recall_at_1 + value: 35.276999999999994 + - type: recall_at_3 + value: 56.739 + - type: recall_at_5 + value: 64.21 + - type: recall_at_10 + value: 74.368 + - type: recall_at_20 + value: 81.888 + - type: recall_at_100 + value: 92.26100000000001 + - type: recall_at_1000 + value: 97.109 + - type: precision_at_1 + value: 51.66 + - type: precision_at_3 + value: 30.843999999999998 + - type: precision_at_5 + value: 21.743000000000002 + - type: precision_at_10 + value: 12.988 + - type: precision_at_20 + value: 7.364999999999999 + - type: precision_at_100 + value: 1.714 + - type: precision_at_1000 + value: 0.184 + - type: mrr_at_1 + value: 51.6598 + - type: mrr_at_3 + value: 60.338899999999995 + - type: mrr_at_5 + value: 61.7808 + - type: mrr_at_10 + value: 62.751599999999996 + - type: mrr_at_20 + value: 63.1412 + - type: mrr_at_100 + value: 63.309099999999994 + - type: mrr_at_1000 + value: 63.317299999999996 + - type: nauc_ndcg_at_1_max + value: 33.6073 + - type: nauc_ndcg_at_1_std + value: 6.1046000000000005 + - type: nauc_ndcg_at_1_diff1 + value: 41.1955 + - type: nauc_ndcg_at_3_max + value: 31.268400000000003 + - type: nauc_ndcg_at_3_std + value: -2.9395000000000002 + - type: nauc_ndcg_at_3_diff1 + value: 35.6186 + - type: nauc_ndcg_at_5_max + value: 32.3145 + - type: nauc_ndcg_at_5_std + value: -0.7283999999999999 + - type: nauc_ndcg_at_5_diff1 + value: 37.7602 + - type: nauc_ndcg_at_10_max + value: 35.1426 + - type: nauc_ndcg_at_10_std + value: -0.13829999999999998 + - type: nauc_ndcg_at_10_diff1 + value: 36.8929 + - type: nauc_ndcg_at_20_max + value: 35.4227 + - type: nauc_ndcg_at_20_std + value: 0.8394999999999999 + - type: nauc_ndcg_at_20_diff1 + value: 36.9758 + - type: nauc_ndcg_at_100_max + value: 36.9415 + - type: nauc_ndcg_at_100_std + value: 5.9117999999999995 + - type: nauc_ndcg_at_100_diff1 + value: 37.0021 + - type: nauc_ndcg_at_1000_max + value: 37.0195 + - type: nauc_ndcg_at_1000_std + value: 5.5642 + - type: nauc_ndcg_at_1000_diff1 + value: 37.1389 + - type: nauc_map_at_1_max + value: 14.893600000000001 + - type: nauc_map_at_1_std + value: -6.9723 + - type: nauc_map_at_1_diff1 + value: 47.328399999999995 + - type: nauc_map_at_3_max + value: 25.1304 + - type: nauc_map_at_3_std + value: -5.5777 + - type: nauc_map_at_3_diff1 + value: 39.5728 + - type: nauc_map_at_5_max + value: 28.206599999999998 + - type: nauc_map_at_5_std + value: -3.2870000000000004 + - type: nauc_map_at_5_diff1 + value: 39.868500000000004 + - type: nauc_map_at_10_max + value: 30.520999999999997 + - type: nauc_map_at_10_std + value: -2.539 + - type: nauc_map_at_10_diff1 + value: 39.1287 + - type: nauc_map_at_20_max + value: 30.712899999999998 + - type: nauc_map_at_20_std + value: -2.0093 + - type: nauc_map_at_20_diff1 + value: 39.0357 + - type: nauc_map_at_100_max + value: 31.0687 + - type: nauc_map_at_100_std + value: -1.0538 + - type: nauc_map_at_100_diff1 + value: 38.9851 + - type: nauc_map_at_1000_max + value: 31.0939 + - type: nauc_map_at_1000_std + value: -1.0348 + - type: nauc_map_at_1000_diff1 + value: 38.9719 + - type: nauc_recall_at_1_max + value: 14.893600000000001 + - type: nauc_recall_at_1_std + value: -6.9723 + - type: nauc_recall_at_1_diff1 + value: 47.328399999999995 + - type: nauc_recall_at_3_max + value: 25.0525 + - type: nauc_recall_at_3_std + value: -9.808300000000001 + - type: nauc_recall_at_3_diff1 + value: 32.9087 + - type: nauc_recall_at_5_max + value: 28.8065 + - type: nauc_recall_at_5_std + value: -4.5512999999999995 + - type: nauc_recall_at_5_diff1 + value: 32.9308 + - type: nauc_recall_at_10_max + value: 34.9121 + - type: nauc_recall_at_10_std + value: -5.8499 + - type: nauc_recall_at_10_diff1 + value: 29.791 + - type: nauc_recall_at_20_max + value: 35.6729 + - type: nauc_recall_at_20_std + value: -4.3512 + - type: nauc_recall_at_20_diff1 + value: 29.087600000000002 + - type: nauc_recall_at_100_max + value: 53.5866 + - type: nauc_recall_at_100_std + value: 49.692 + - type: nauc_recall_at_100_diff1 + value: 28.9725 + - type: nauc_recall_at_1000_max + value: 80.23949999999999 + - type: nauc_recall_at_1000_std + value: 86.7359 + - type: nauc_recall_at_1000_diff1 + value: 37.333 + - type: nauc_precision_at_1_max + value: 33.6073 + - type: nauc_precision_at_1_std + value: 6.1046000000000005 + - type: nauc_precision_at_1_diff1 + value: 41.1955 + - type: nauc_precision_at_3_max + value: 40.2515 + - type: nauc_precision_at_3_std + value: 12.1973 + - type: nauc_precision_at_3_diff1 + value: 3.9177999999999997 + - type: nauc_precision_at_5_max + value: 41.7312 + - type: nauc_precision_at_5_std + value: 17.921400000000002 + - type: nauc_precision_at_5_diff1 + value: -0.2405 + - type: nauc_precision_at_10_max + value: 39.9025 + - type: nauc_precision_at_10_std + value: 18.9909 + - type: nauc_precision_at_10_diff1 + value: -8.5406 + - type: nauc_precision_at_20_max + value: 34.1753 + - type: nauc_precision_at_20_std + value: 21.9853 + - type: nauc_precision_at_20_diff1 + value: -13.966700000000001 + - type: nauc_precision_at_100_max + value: 30.461 + - type: nauc_precision_at_100_std + value: 34.063900000000004 + - type: nauc_precision_at_100_diff1 + value: -21.1252 + - type: nauc_precision_at_1000_max + value: 26.5512 + - type: nauc_precision_at_1000_std + value: 30.7066 + - type: nauc_precision_at_1000_diff1 + value: -22.2902 + - type: nauc_mrr_at_1_max + value: 33.6073 + - type: nauc_mrr_at_1_std + value: 6.1046000000000005 + - type: nauc_mrr_at_1_diff1 + value: 41.1955 + - type: nauc_mrr_at_3_max + value: 37.6571 + - type: nauc_mrr_at_3_std + value: 5.2793 + - type: nauc_mrr_at_3_diff1 + value: 36.5302 + - type: nauc_mrr_at_5_max + value: 38.6239 + - type: nauc_mrr_at_5_std + value: 7.762700000000001 + - type: nauc_mrr_at_5_diff1 + value: 36.525 + - type: nauc_mrr_at_10_max + value: 38.4608 + - type: nauc_mrr_at_10_std + value: 7.131 + - type: nauc_mrr_at_10_diff1 + value: 36.4653 + - type: nauc_mrr_at_20_max + value: 38.2783 + - type: nauc_mrr_at_20_std + value: 6.9415000000000004 + - type: nauc_mrr_at_20_diff1 + value: 36.5089 + - type: nauc_mrr_at_100_max + value: 38.337199999999996 + - type: nauc_mrr_at_100_std + value: 7.2228 + - type: nauc_mrr_at_100_diff1 + value: 36.6891 + - type: nauc_mrr_at_1000_max + value: 38.327600000000004 + - type: nauc_mrr_at_1000_std + value: 7.206300000000001 + - type: nauc_mrr_at_1000_diff1 + value: 36.696400000000004 + - type: main_score + value: 61.271 + task: + type: Retrieval + - dataset: + config: te + name: MTEB MIRACLRetrieval (te) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 63.647 + - type: ndcg_at_3 + value: 75.98700000000001 + - type: ndcg_at_5 + value: 77.86999999999999 + - type: ndcg_at_10 + value: 79.149 + - type: ndcg_at_20 + value: 79.50399999999999 + - type: ndcg_at_100 + value: 80.199 + - type: ndcg_at_1000 + value: 80.393 + - type: map_at_1 + value: 62.963 + - type: map_at_3 + value: 72.94699999999999 + - type: map_at_5 + value: 74.042 + - type: map_at_10 + value: 74.612 + - type: map_at_20 + value: 74.727 + - type: map_at_100 + value: 74.831 + - type: map_at_1000 + value: 74.839 + - type: recall_at_1 + value: 62.963 + - type: recall_at_3 + value: 84.15899999999999 + - type: recall_at_5 + value: 88.627 + - type: recall_at_10 + value: 92.411 + - type: recall_at_20 + value: 93.74 + - type: recall_at_100 + value: 97.363 + - type: recall_at_1000 + value: 98.833 + - type: precision_at_1 + value: 63.647 + - type: precision_at_3 + value: 28.622999999999998 + - type: precision_at_5 + value: 18.163999999999998 + - type: precision_at_10 + value: 9.481 + - type: precision_at_20 + value: 4.819 + - type: precision_at_100 + value: 1.001 + - type: precision_at_1000 + value: 0.10200000000000001 + - type: mrr_at_1 + value: 63.647299999999994 + - type: mrr_at_3 + value: 73.49029999999999 + - type: mrr_at_5 + value: 74.4626 + - type: mrr_at_10 + value: 74.98280000000001 + - type: mrr_at_20 + value: 75.0719 + - type: mrr_at_100 + value: 75.1695 + - type: mrr_at_1000 + value: 75.1769 + - type: nauc_ndcg_at_1_max + value: 33.3063 + - type: nauc_ndcg_at_1_std + value: -27.609699999999997 + - type: nauc_ndcg_at_1_diff1 + value: 64.8293 + - type: nauc_ndcg_at_3_max + value: 42.4738 + - type: nauc_ndcg_at_3_std + value: -23.8921 + - type: nauc_ndcg_at_3_diff1 + value: 56.43749999999999 + - type: nauc_ndcg_at_5_max + value: 43.132 + - type: nauc_ndcg_at_5_std + value: -23.2181 + - type: nauc_ndcg_at_5_diff1 + value: 55.722899999999996 + - type: nauc_ndcg_at_10_max + value: 43.036 + - type: nauc_ndcg_at_10_std + value: -22.880300000000002 + - type: nauc_ndcg_at_10_diff1 + value: 56.22279999999999 + - type: nauc_ndcg_at_20_max + value: 43.1538 + - type: nauc_ndcg_at_20_std + value: -22.7674 + - type: nauc_ndcg_at_20_diff1 + value: 56.4893 + - type: nauc_ndcg_at_100_max + value: 42.0908 + - type: nauc_ndcg_at_100_std + value: -22.3071 + - type: nauc_ndcg_at_100_diff1 + value: 57.5928 + - type: nauc_ndcg_at_1000_max + value: 41.6223 + - type: nauc_ndcg_at_1000_std + value: -22.747600000000002 + - type: nauc_ndcg_at_1000_diff1 + value: 57.6603 + - type: nauc_map_at_1_max + value: 31.9355 + - type: nauc_map_at_1_std + value: -29.4362 + - type: nauc_map_at_1_diff1 + value: 64.9802 + - type: nauc_map_at_3_max + value: 39.3304 + - type: nauc_map_at_3_std + value: -25.819 + - type: nauc_map_at_3_diff1 + value: 58.8664 + - type: nauc_map_at_5_max + value: 39.659800000000004 + - type: nauc_map_at_5_std + value: -25.3619 + - type: nauc_map_at_5_diff1 + value: 58.57449999999999 + - type: nauc_map_at_10_max + value: 39.6121 + - type: nauc_map_at_10_std + value: -25.2399 + - type: nauc_map_at_10_diff1 + value: 58.8083 + - type: nauc_map_at_20_max + value: 39.6958 + - type: nauc_map_at_20_std + value: -25.116 + - type: nauc_map_at_20_diff1 + value: 58.8995 + - type: nauc_map_at_100_max + value: 39.5617 + - type: nauc_map_at_100_std + value: -25.0319 + - type: nauc_map_at_100_diff1 + value: 59.053599999999996 + - type: nauc_map_at_1000_max + value: 39.5469 + - type: nauc_map_at_1000_std + value: -25.0473 + - type: nauc_map_at_1000_diff1 + value: 59.0556 + - type: nauc_recall_at_1_max + value: 31.9355 + - type: nauc_recall_at_1_std + value: -29.4362 + - type: nauc_recall_at_1_diff1 + value: 64.9802 + - type: nauc_recall_at_3_max + value: 54.57149999999999 + - type: nauc_recall_at_3_std + value: -17.9671 + - type: nauc_recall_at_3_diff1 + value: 45.4961 + - type: nauc_recall_at_5_max + value: 61.2002 + - type: nauc_recall_at_5_std + value: -13.9075 + - type: nauc_recall_at_5_diff1 + value: 39.1115 + - type: nauc_recall_at_10_max + value: 68.2226 + - type: nauc_recall_at_10_std + value: -7.230200000000001 + - type: nauc_recall_at_10_diff1 + value: 34.9241 + - type: nauc_recall_at_20_max + value: 74.08019999999999 + - type: nauc_recall_at_20_std + value: -4.4287 + - type: nauc_recall_at_20_diff1 + value: 33.4441 + - type: nauc_recall_at_100_max + value: 80.2462 + - type: nauc_recall_at_100_std + value: 30.9842 + - type: nauc_recall_at_100_diff1 + value: 38.0659 + - type: nauc_recall_at_1000_max + value: 77.5197 + - type: nauc_recall_at_1000_std + value: 51.5945 + - type: nauc_recall_at_1000_diff1 + value: 22.9724 + - type: nauc_precision_at_1_max + value: 33.3063 + - type: nauc_precision_at_1_std + value: -27.609699999999997 + - type: nauc_precision_at_1_diff1 + value: 64.8293 + - type: nauc_precision_at_3_max + value: 56.837199999999996 + - type: nauc_precision_at_3_std + value: -7.5578 + - type: nauc_precision_at_3_diff1 + value: 36.4516 + - type: nauc_precision_at_5_max + value: 57.3511 + - type: nauc_precision_at_5_std + value: 2.889 + - type: nauc_precision_at_5_diff1 + value: 23.0276 + - type: nauc_precision_at_10_max + value: 56.852999999999994 + - type: nauc_precision_at_10_std + value: 13.305900000000001 + - type: nauc_precision_at_10_diff1 + value: 12.1547 + - type: nauc_precision_at_20_max + value: 55.735299999999995 + - type: nauc_precision_at_20_std + value: 20.3483 + - type: nauc_precision_at_20_diff1 + value: 6.6423 + - type: nauc_precision_at_100_max + value: 43.358999999999995 + - type: nauc_precision_at_100_std + value: 44.4213 + - type: nauc_precision_at_100_diff1 + value: -5.556500000000001 + - type: nauc_precision_at_1000_max + value: 27.974 + - type: nauc_precision_at_1000_std + value: 47.254400000000004 + - type: nauc_precision_at_1000_diff1 + value: -21.8157 + - type: nauc_mrr_at_1_max + value: 33.3063 + - type: nauc_mrr_at_1_std + value: -27.609699999999997 + - type: nauc_mrr_at_1_diff1 + value: 64.8293 + - type: nauc_mrr_at_3_max + value: 40.129 + - type: nauc_mrr_at_3_std + value: -24.0152 + - type: nauc_mrr_at_3_diff1 + value: 58.9134 + - type: nauc_mrr_at_5_max + value: 40.1054 + - type: nauc_mrr_at_5_std + value: -24.0554 + - type: nauc_mrr_at_5_diff1 + value: 58.71920000000001 + - type: nauc_mrr_at_10_max + value: 40.0067 + - type: nauc_mrr_at_10_std + value: -23.9912 + - type: nauc_mrr_at_10_diff1 + value: 58.964099999999995 + - type: nauc_mrr_at_20_max + value: 39.9983 + - type: nauc_mrr_at_20_std + value: -24.0277 + - type: nauc_mrr_at_20_diff1 + value: 59.0425 + - type: nauc_mrr_at_100_max + value: 39.8766 + - type: nauc_mrr_at_100_std + value: -23.9296 + - type: nauc_mrr_at_100_diff1 + value: 59.1824 + - type: nauc_mrr_at_1000_max + value: 39.861799999999995 + - type: nauc_mrr_at_1000_std + value: -23.9468 + - type: nauc_mrr_at_1000_diff1 + value: 59.1847 + - type: main_score + value: 79.149 + task: + type: Retrieval + - dataset: + config: th + name: MTEB MIRACLRetrieval (th) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 66.712 + - type: ndcg_at_3 + value: 67.393 + - type: ndcg_at_5 + value: 70.20100000000001 + - type: ndcg_at_10 + value: 73.324 + - type: ndcg_at_20 + value: 75.24300000000001 + - type: ndcg_at_100 + value: 76.633 + - type: ndcg_at_1000 + value: 77.119 + - type: map_at_1 + value: 47.105999999999995 + - type: map_at_3 + value: 60.67700000000001 + - type: map_at_5 + value: 63.81099999999999 + - type: map_at_10 + value: 65.998 + - type: map_at_20 + value: 66.914 + - type: map_at_100 + value: 67.258 + - type: map_at_1000 + value: 67.293 + - type: recall_at_1 + value: 47.105999999999995 + - type: recall_at_3 + value: 68.45599999999999 + - type: recall_at_5 + value: 75.91499999999999 + - type: recall_at_10 + value: 84.294 + - type: recall_at_20 + value: 90.08500000000001 + - type: recall_at_100 + value: 95.949 + - type: recall_at_1000 + value: 98.874 + - type: precision_at_1 + value: 66.712 + - type: precision_at_3 + value: 36.016 + - type: precision_at_5 + value: 25.157 + - type: precision_at_10 + value: 14.516000000000002 + - type: precision_at_20 + value: 7.994999999999999 + - type: precision_at_100 + value: 1.738 + - type: precision_at_1000 + value: 0.181 + - type: mrr_at_1 + value: 66.71209999999999 + - type: mrr_at_3 + value: 74.3747 + - type: mrr_at_5 + value: 75.3297 + - type: mrr_at_10 + value: 75.9858 + - type: mrr_at_20 + value: 76.1819 + - type: mrr_at_100 + value: 76.2551 + - type: mrr_at_1000 + value: 76.2587 + - type: nauc_ndcg_at_1_max + value: 43.199799999999996 + - type: nauc_ndcg_at_1_std + value: 8.6242 + - type: nauc_ndcg_at_1_diff1 + value: 49.3688 + - type: nauc_ndcg_at_3_max + value: 37.9248 + - type: nauc_ndcg_at_3_std + value: -1.3769 + - type: nauc_ndcg_at_3_diff1 + value: 39.9588 + - type: nauc_ndcg_at_5_max + value: 38.4241 + - type: nauc_ndcg_at_5_std + value: -1.0533000000000001 + - type: nauc_ndcg_at_5_diff1 + value: 40.0453 + - type: nauc_ndcg_at_10_max + value: 40.4105 + - type: nauc_ndcg_at_10_std + value: 1.4455 + - type: nauc_ndcg_at_10_diff1 + value: 40.6256 + - type: nauc_ndcg_at_20_max + value: 41.1133 + - type: nauc_ndcg_at_20_std + value: 2.931 + - type: nauc_ndcg_at_20_diff1 + value: 40.920899999999996 + - type: nauc_ndcg_at_100_max + value: 41.6336 + - type: nauc_ndcg_at_100_std + value: 4.9768 + - type: nauc_ndcg_at_100_diff1 + value: 41.3658 + - type: nauc_ndcg_at_1000_max + value: 41.6223 + - type: nauc_ndcg_at_1000_std + value: 5.2031 + - type: nauc_ndcg_at_1000_diff1 + value: 41.4062 + - type: nauc_map_at_1_max + value: 20.7626 + - type: nauc_map_at_1_std + value: -8.0023 + - type: nauc_map_at_1_diff1 + value: 44.4569 + - type: nauc_map_at_3_max + value: 32.5175 + - type: nauc_map_at_3_std + value: -7.458099999999999 + - type: nauc_map_at_3_diff1 + value: 40.2164 + - type: nauc_map_at_5_max + value: 34.4803 + - type: nauc_map_at_5_std + value: -5.149 + - type: nauc_map_at_5_diff1 + value: 39.7814 + - type: nauc_map_at_10_max + value: 36.0112 + - type: nauc_map_at_10_std + value: -2.7143 + - type: nauc_map_at_10_diff1 + value: 40.231 + - type: nauc_map_at_20_max + value: 36.574200000000005 + - type: nauc_map_at_20_std + value: -1.718 + - type: nauc_map_at_20_diff1 + value: 40.278000000000006 + - type: nauc_map_at_100_max + value: 36.7445 + - type: nauc_map_at_100_std + value: -1.208 + - type: nauc_map_at_100_diff1 + value: 40.4046 + - type: nauc_map_at_1000_max + value: 36.770199999999996 + - type: nauc_map_at_1000_std + value: -1.1672 + - type: nauc_map_at_1000_diff1 + value: 40.409099999999995 + - type: nauc_recall_at_1_max + value: 20.7626 + - type: nauc_recall_at_1_std + value: -8.0023 + - type: nauc_recall_at_1_diff1 + value: 44.4569 + - type: nauc_recall_at_3_max + value: 31.2938 + - type: nauc_recall_at_3_std + value: -12.4723 + - type: nauc_recall_at_3_diff1 + value: 35.0524 + - type: nauc_recall_at_5_max + value: 34.4221 + - type: nauc_recall_at_5_std + value: -9.0849 + - type: nauc_recall_at_5_diff1 + value: 33.6966 + - type: nauc_recall_at_10_max + value: 40.1481 + - type: nauc_recall_at_10_std + value: -2.4007 + - type: nauc_recall_at_10_diff1 + value: 32.398700000000005 + - type: nauc_recall_at_20_max + value: 43.068400000000004 + - type: nauc_recall_at_20_std + value: 0.4869 + - type: nauc_recall_at_20_diff1 + value: 31.7169 + - type: nauc_recall_at_100_max + value: 54.1481 + - type: nauc_recall_at_100_std + value: 28.3243 + - type: nauc_recall_at_100_diff1 + value: 29.1055 + - type: nauc_recall_at_1000_max + value: 82.51389999999999 + - type: nauc_recall_at_1000_std + value: 88.3602 + - type: nauc_recall_at_1000_diff1 + value: 14.9201 + - type: nauc_precision_at_1_max + value: 43.199799999999996 + - type: nauc_precision_at_1_std + value: 8.6242 + - type: nauc_precision_at_1_diff1 + value: 49.3688 + - type: nauc_precision_at_3_max + value: 35.1732 + - type: nauc_precision_at_3_std + value: 16.3941 + - type: nauc_precision_at_3_diff1 + value: 4.4193999999999996 + - type: nauc_precision_at_5_max + value: 28.2059 + - type: nauc_precision_at_5_std + value: 22.4744 + - type: nauc_precision_at_5_diff1 + value: -4.0808 + - type: nauc_precision_at_10_max + value: 22.7955 + - type: nauc_precision_at_10_std + value: 28.8744 + - type: nauc_precision_at_10_diff1 + value: -9.9309 + - type: nauc_precision_at_20_max + value: 17.2362 + - type: nauc_precision_at_20_std + value: 30.7132 + - type: nauc_precision_at_20_diff1 + value: -13.5708 + - type: nauc_precision_at_100_max + value: 13.3455 + - type: nauc_precision_at_100_std + value: 34.1715 + - type: nauc_precision_at_100_diff1 + value: -16.4298 + - type: nauc_precision_at_1000_max + value: 10.639700000000001 + - type: nauc_precision_at_1000_std + value: 33.1325 + - type: nauc_precision_at_1000_diff1 + value: -17.5938 + - type: nauc_mrr_at_1_max + value: 43.199799999999996 + - type: nauc_mrr_at_1_std + value: 8.6242 + - type: nauc_mrr_at_1_diff1 + value: 49.3688 + - type: nauc_mrr_at_3_max + value: 47.106500000000004 + - type: nauc_mrr_at_3_std + value: 10.3023 + - type: nauc_mrr_at_3_diff1 + value: 46.2565 + - type: nauc_mrr_at_5_max + value: 47.151900000000005 + - type: nauc_mrr_at_5_std + value: 11.2485 + - type: nauc_mrr_at_5_diff1 + value: 46.4519 + - type: nauc_mrr_at_10_max + value: 47.468700000000005 + - type: nauc_mrr_at_10_std + value: 11.5245 + - type: nauc_mrr_at_10_diff1 + value: 46.291399999999996 + - type: nauc_mrr_at_20_max + value: 47.3577 + - type: nauc_mrr_at_20_std + value: 11.3081 + - type: nauc_mrr_at_20_diff1 + value: 46.490700000000004 + - type: nauc_mrr_at_100_max + value: 47.3153 + - type: nauc_mrr_at_100_std + value: 11.2816 + - type: nauc_mrr_at_100_diff1 + value: 46.5288 + - type: nauc_mrr_at_1000_max + value: 47.308299999999996 + - type: nauc_mrr_at_1000_std + value: 11.2835 + - type: nauc_mrr_at_1000_diff1 + value: 46.5276 + - type: main_score + value: 73.324 + task: + type: Retrieval + - dataset: + config: yo + name: MTEB MIRACLRetrieval (yo) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 49.58 + - type: ndcg_at_3 + value: 64.793 + - type: ndcg_at_5 + value: 66.709 + - type: ndcg_at_10 + value: 68.705 + - type: ndcg_at_20 + value: 69.8 + - type: ndcg_at_100 + value: 70.664 + - type: ndcg_at_1000 + value: 71.197 + - type: map_at_1 + value: 46.289 + - type: map_at_3 + value: 59.921 + - type: map_at_5 + value: 61.409000000000006 + - type: map_at_10 + value: 62.379 + - type: map_at_20 + value: 62.773 + - type: map_at_100 + value: 62.907000000000004 + - type: map_at_1000 + value: 62.922999999999995 + - type: recall_at_1 + value: 46.289 + - type: recall_at_3 + value: 75.07000000000001 + - type: recall_at_5 + value: 79.202 + - type: recall_at_10 + value: 85.154 + - type: recall_at_20 + value: 89.076 + - type: recall_at_100 + value: 93.557 + - type: recall_at_1000 + value: 97.479 + - type: precision_at_1 + value: 49.58 + - type: precision_at_3 + value: 28.571 + - type: precision_at_5 + value: 18.655 + - type: precision_at_10 + value: 10.084 + - type: precision_at_20 + value: 5.2940000000000005 + - type: precision_at_100 + value: 1.109 + - type: precision_at_1000 + value: 0.11800000000000001 + - type: mrr_at_1 + value: 49.5798 + - type: mrr_at_3 + value: 63.025200000000005 + - type: mrr_at_5 + value: 63.6134 + - type: mrr_at_10 + value: 64.2504 + - type: mrr_at_20 + value: 64.5152 + - type: mrr_at_100 + value: 64.6281 + - type: mrr_at_1000 + value: 64.63839999999999 + - type: nauc_ndcg_at_1_max + value: 18.5119 + - type: nauc_ndcg_at_1_std + value: -26.7799 + - type: nauc_ndcg_at_1_diff1 + value: 49.55 + - type: nauc_ndcg_at_3_max + value: 35.6833 + - type: nauc_ndcg_at_3_std + value: -19.023699999999998 + - type: nauc_ndcg_at_3_diff1 + value: 51.4553 + - type: nauc_ndcg_at_5_max + value: 34.252700000000004 + - type: nauc_ndcg_at_5_std + value: -16.9909 + - type: nauc_ndcg_at_5_diff1 + value: 50.034 + - type: nauc_ndcg_at_10_max + value: 35.115899999999996 + - type: nauc_ndcg_at_10_std + value: -15.454300000000002 + - type: nauc_ndcg_at_10_diff1 + value: 51.13419999999999 + - type: nauc_ndcg_at_20_max + value: 36.3127 + - type: nauc_ndcg_at_20_std + value: -13.5123 + - type: nauc_ndcg_at_20_diff1 + value: 52.505100000000006 + - type: nauc_ndcg_at_100_max + value: 35.0788 + - type: nauc_ndcg_at_100_std + value: -15.118 + - type: nauc_ndcg_at_100_diff1 + value: 52.2994 + - type: nauc_ndcg_at_1000_max + value: 34.1448 + - type: nauc_ndcg_at_1000_std + value: -15.695300000000001 + - type: nauc_ndcg_at_1000_diff1 + value: 51.7561 + - type: nauc_map_at_1_max + value: 17.9766 + - type: nauc_map_at_1_std + value: -26.0689 + - type: nauc_map_at_1_diff1 + value: 51.3004 + - type: nauc_map_at_3_max + value: 30.426 + - type: nauc_map_at_3_std + value: -21.5618 + - type: nauc_map_at_3_diff1 + value: 51.9665 + - type: nauc_map_at_5_max + value: 30.3093 + - type: nauc_map_at_5_std + value: -19.1582 + - type: nauc_map_at_5_diff1 + value: 50.9919 + - type: nauc_map_at_10_max + value: 31.1197 + - type: nauc_map_at_10_std + value: -18.5626 + - type: nauc_map_at_10_diff1 + value: 51.3278 + - type: nauc_map_at_20_max + value: 31.3984 + - type: nauc_map_at_20_std + value: -17.8214 + - type: nauc_map_at_20_diff1 + value: 51.5951 + - type: nauc_map_at_100_max + value: 31.1974 + - type: nauc_map_at_100_std + value: -18.0483 + - type: nauc_map_at_100_diff1 + value: 51.51559999999999 + - type: nauc_map_at_1000_max + value: 31.167699999999996 + - type: nauc_map_at_1000_std + value: -18.076800000000002 + - type: nauc_map_at_1000_diff1 + value: 51.50130000000001 + - type: nauc_recall_at_1_max + value: 17.9766 + - type: nauc_recall_at_1_std + value: -26.0689 + - type: nauc_recall_at_1_diff1 + value: 51.3004 + - type: nauc_recall_at_3_max + value: 48.720200000000006 + - type: nauc_recall_at_3_std + value: -12.1143 + - type: nauc_recall_at_3_diff1 + value: 49.863800000000005 + - type: nauc_recall_at_5_max + value: 48.1997 + - type: nauc_recall_at_5_std + value: -5.8457 + - type: nauc_recall_at_5_diff1 + value: 46.062599999999996 + - type: nauc_recall_at_10_max + value: 56.5698 + - type: nauc_recall_at_10_std + value: 6.0906 + - type: nauc_recall_at_10_diff1 + value: 51.9053 + - type: nauc_recall_at_20_max + value: 73.61569999999999 + - type: nauc_recall_at_20_std + value: 25.8535 + - type: nauc_recall_at_20_diff1 + value: 64.7516 + - type: nauc_recall_at_100_max + value: 78.054 + - type: nauc_recall_at_100_std + value: 23.7984 + - type: nauc_recall_at_100_diff1 + value: 71.61999999999999 + - type: nauc_recall_at_1000_max + value: 92.5519 + - type: nauc_recall_at_1000_std + value: 59.609100000000005 + - type: nauc_recall_at_1000_diff1 + value: 78.6415 + - type: nauc_precision_at_1_max + value: 18.5119 + - type: nauc_precision_at_1_std + value: -26.7799 + - type: nauc_precision_at_1_diff1 + value: 49.55 + - type: nauc_precision_at_3_max + value: 45.402100000000004 + - type: nauc_precision_at_3_std + value: -5.331 + - type: nauc_precision_at_3_diff1 + value: 20.6481 + - type: nauc_precision_at_5_max + value: 33.7262 + - type: nauc_precision_at_5_std + value: 10.3483 + - type: nauc_precision_at_5_diff1 + value: 5.9393 + - type: nauc_precision_at_10_max + value: 35.3715 + - type: nauc_precision_at_10_std + value: 17.0809 + - type: nauc_precision_at_10_diff1 + value: 0.9325 + - type: nauc_precision_at_20_max + value: 35.2666 + - type: nauc_precision_at_20_std + value: 26.3214 + - type: nauc_precision_at_20_diff1 + value: -1.8064 + - type: nauc_precision_at_100_max + value: 29.0385 + - type: nauc_precision_at_100_std + value: 23.416500000000003 + - type: nauc_precision_at_100_diff1 + value: -10.83 + - type: nauc_precision_at_1000_max + value: 13.825299999999999 + - type: nauc_precision_at_1000_std + value: 16.7663 + - type: nauc_precision_at_1000_diff1 + value: -24.854200000000002 + - type: nauc_mrr_at_1_max + value: 18.5119 + - type: nauc_mrr_at_1_std + value: -26.7799 + - type: nauc_mrr_at_1_diff1 + value: 49.55 + - type: nauc_mrr_at_3_max + value: 29.916500000000003 + - type: nauc_mrr_at_3_std + value: -21.5719 + - type: nauc_mrr_at_3_diff1 + value: 50.2057 + - type: nauc_mrr_at_5_max + value: 28.929 + - type: nauc_mrr_at_5_std + value: -21.9015 + - type: nauc_mrr_at_5_diff1 + value: 49.6675 + - type: nauc_mrr_at_10_max + value: 28.6377 + - type: nauc_mrr_at_10_std + value: -21.4266 + - type: nauc_mrr_at_10_diff1 + value: 50.034800000000004 + - type: nauc_mrr_at_20_max + value: 28.7905 + - type: nauc_mrr_at_20_std + value: -21.192 + - type: nauc_mrr_at_20_diff1 + value: 50.3745 + - type: nauc_mrr_at_100_max + value: 28.5717 + - type: nauc_mrr_at_100_std + value: -21.3735 + - type: nauc_mrr_at_100_diff1 + value: 50.3333 + - type: nauc_mrr_at_1000_max + value: 28.5655 + - type: nauc_mrr_at_1000_std + value: -21.373 + - type: nauc_mrr_at_1000_diff1 + value: 50.3215 + - type: main_score + value: 68.705 + task: + type: Retrieval + - dataset: + config: zh + name: MTEB MIRACLRetrieval (zh) + revision: main + split: dev + type: miracl/mmteb-miracl + metrics: + - type: ndcg_at_1 + value: 47.583 + - type: ndcg_at_3 + value: 45.839 + - type: ndcg_at_5 + value: 48.126999999999995 + - type: ndcg_at_10 + value: 52.553000000000004 + - type: ndcg_at_20 + value: 55.66799999999999 + - type: ndcg_at_100 + value: 60.0 + - type: ndcg_at_1000 + value: 61.415 + - type: map_at_1 + value: 24.488 + - type: map_at_3 + value: 36.202 + - type: map_at_5 + value: 39.771 + - type: map_at_10 + value: 42.725 + - type: map_at_20 + value: 44.163999999999994 + - type: map_at_100 + value: 45.269 + - type: map_at_1000 + value: 45.372 + - type: recall_at_1 + value: 24.488 + - type: recall_at_3 + value: 42.827 + - type: recall_at_5 + value: 52.081 + - type: recall_at_10 + value: 63.659 + - type: recall_at_20 + value: 72.652 + - type: recall_at_100 + value: 89.702 + - type: recall_at_1000 + value: 97.99600000000001 + - type: precision_at_1 + value: 47.583 + - type: precision_at_3 + value: 30.789 + - type: precision_at_5 + value: 23.206 + - type: precision_at_10 + value: 14.885000000000002 + - type: precision_at_20 + value: 8.803999999999998 + - type: precision_at_100 + value: 2.237 + - type: precision_at_1000 + value: 0.247 + - type: mrr_at_1 + value: 47.5827 + - type: mrr_at_3 + value: 56.4461 + - type: mrr_at_5 + value: 58.036500000000004 + - type: mrr_at_10 + value: 59.2419 + - type: mrr_at_20 + value: 59.5684 + - type: mrr_at_100 + value: 59.8496 + - type: mrr_at_1000 + value: 59.868500000000004 + - type: nauc_ndcg_at_1_max + value: 30.3153 + - type: nauc_ndcg_at_1_std + value: 16.1917 + - type: nauc_ndcg_at_1_diff1 + value: 33.1291 + - type: nauc_ndcg_at_3_max + value: 29.9473 + - type: nauc_ndcg_at_3_std + value: 9.9602 + - type: nauc_ndcg_at_3_diff1 + value: 26.354899999999997 + - type: nauc_ndcg_at_5_max + value: 27.5364 + - type: nauc_ndcg_at_5_std + value: 9.0106 + - type: nauc_ndcg_at_5_diff1 + value: 26.4299 + - type: nauc_ndcg_at_10_max + value: 30.1141 + - type: nauc_ndcg_at_10_std + value: 10.6319 + - type: nauc_ndcg_at_10_diff1 + value: 26.1015 + - type: nauc_ndcg_at_20_max + value: 31.864700000000003 + - type: nauc_ndcg_at_20_std + value: 14.376 + - type: nauc_ndcg_at_20_diff1 + value: 24.278 + - type: nauc_ndcg_at_100_max + value: 33.8328 + - type: nauc_ndcg_at_100_std + value: 17.1646 + - type: nauc_ndcg_at_100_diff1 + value: 24.7582 + - type: nauc_ndcg_at_1000_max + value: 33.0653 + - type: nauc_ndcg_at_1000_std + value: 15.717400000000001 + - type: nauc_ndcg_at_1000_diff1 + value: 25.708399999999997 + - type: nauc_map_at_1_max + value: 14.5636 + - type: nauc_map_at_1_std + value: -0.5065 + - type: nauc_map_at_1_diff1 + value: 37.5816 + - type: nauc_map_at_3_max + value: 21.752 + - type: nauc_map_at_3_std + value: 0.2942 + - type: nauc_map_at_3_diff1 + value: 29.662100000000002 + - type: nauc_map_at_5_max + value: 23.3994 + - type: nauc_map_at_5_std + value: 3.2369000000000003 + - type: nauc_map_at_5_diff1 + value: 28.479 + - type: nauc_map_at_10_max + value: 26.969500000000004 + - type: nauc_map_at_10_std + value: 6.4338999999999995 + - type: nauc_map_at_10_diff1 + value: 27.548000000000002 + - type: nauc_map_at_20_max + value: 28.2804 + - type: nauc_map_at_20_std + value: 8.3557 + - type: nauc_map_at_20_diff1 + value: 26.561600000000002 + - type: nauc_map_at_100_max + value: 28.979899999999997 + - type: nauc_map_at_100_std + value: 9.3446 + - type: nauc_map_at_100_diff1 + value: 26.539099999999998 + - type: nauc_map_at_1000_max + value: 28.9572 + - type: nauc_map_at_1000_std + value: 9.3017 + - type: nauc_map_at_1000_diff1 + value: 26.6029 + - type: nauc_recall_at_1_max + value: 14.5636 + - type: nauc_recall_at_1_std + value: -0.5065 + - type: nauc_recall_at_1_diff1 + value: 37.5816 + - type: nauc_recall_at_3_max + value: 19.8958 + - type: nauc_recall_at_3_std + value: -1.7080000000000002 + - type: nauc_recall_at_3_diff1 + value: 24.4885 + - type: nauc_recall_at_5_max + value: 18.8426 + - type: nauc_recall_at_5_std + value: 3.5769 + - type: nauc_recall_at_5_diff1 + value: 21.253700000000002 + - type: nauc_recall_at_10_max + value: 25.061299999999996 + - type: nauc_recall_at_10_std + value: 7.1753 + - type: nauc_recall_at_10_diff1 + value: 18.7378 + - type: nauc_recall_at_20_max + value: 28.6096 + - type: nauc_recall_at_20_std + value: 18.5789 + - type: nauc_recall_at_20_diff1 + value: 11.686 + - type: nauc_recall_at_100_max + value: 45.903 + - type: nauc_recall_at_100_std + value: 46.9916 + - type: nauc_recall_at_100_diff1 + value: 9.813600000000001 + - type: nauc_recall_at_1000_max + value: 62.512699999999995 + - type: nauc_recall_at_1000_std + value: 67.9442 + - type: nauc_recall_at_1000_diff1 + value: 34.3912 + - type: nauc_precision_at_1_max + value: 30.3153 + - type: nauc_precision_at_1_std + value: 16.1917 + - type: nauc_precision_at_1_diff1 + value: 33.1291 + - type: nauc_precision_at_3_max + value: 35.6697 + - type: nauc_precision_at_3_std + value: 18.0247 + - type: nauc_precision_at_3_diff1 + value: 7.0163 + - type: nauc_precision_at_5_max + value: 34.0555 + - type: nauc_precision_at_5_std + value: 23.5324 + - type: nauc_precision_at_5_diff1 + value: 0.44270000000000004 + - type: nauc_precision_at_10_max + value: 37.8515 + - type: nauc_precision_at_10_std + value: 31.657000000000004 + - type: nauc_precision_at_10_diff1 + value: -5.2642 + - type: nauc_precision_at_20_max + value: 36.025 + - type: nauc_precision_at_20_std + value: 35.236000000000004 + - type: nauc_precision_at_20_diff1 + value: -10.6916 + - type: nauc_precision_at_100_max + value: 29.678900000000002 + - type: nauc_precision_at_100_std + value: 35.2162 + - type: nauc_precision_at_100_diff1 + value: -13.7845 + - type: nauc_precision_at_1000_max + value: 22.2855 + - type: nauc_precision_at_1000_std + value: 27.221600000000002 + - type: nauc_precision_at_1000_diff1 + value: -13.4482 + - type: nauc_mrr_at_1_max + value: 30.3153 + - type: nauc_mrr_at_1_std + value: 16.1917 + - type: nauc_mrr_at_1_diff1 + value: 33.1291 + - type: nauc_mrr_at_3_max + value: 33.2966 + - type: nauc_mrr_at_3_std + value: 16.9755 + - type: nauc_mrr_at_3_diff1 + value: 29.814 + - type: nauc_mrr_at_5_max + value: 32.920300000000005 + - type: nauc_mrr_at_5_std + value: 17.832600000000003 + - type: nauc_mrr_at_5_diff1 + value: 29.683300000000003 + - type: nauc_mrr_at_10_max + value: 32.9394 + - type: nauc_mrr_at_10_std + value: 17.5036 + - type: nauc_mrr_at_10_diff1 + value: 29.6425 + - type: nauc_mrr_at_20_max + value: 32.852599999999995 + - type: nauc_mrr_at_20_std + value: 17.8307 + - type: nauc_mrr_at_20_diff1 + value: 29.4502 + - type: nauc_mrr_at_100_max + value: 32.9242 + - type: nauc_mrr_at_100_std + value: 17.7699 + - type: nauc_mrr_at_100_diff1 + value: 29.504399999999997 + - type: nauc_mrr_at_1000_max + value: 32.9303 + - type: nauc_mrr_at_1000_std + value: 17.7636 + - type: nauc_mrr_at_1000_diff1 + value: 29.526799999999998 + - type: main_score + value: 52.553000000000004 + task: + type: Retrieval + - dataset: + config: default + name: MTEB MSMARCO (default) + revision: c5a29a104738b98a9e76336939199e264163d4a0 + split: dev + type: mteb/msmarco + metrics: + - type: ndcg_at_1 + value: 14.155000000000001 + - type: ndcg_at_3 + value: 22.499 + - type: ndcg_at_5 + value: 26.233 + - type: ndcg_at_10 + value: 29.866999999999997 + - type: ndcg_at_20 + value: 32.616 + - type: ndcg_at_100 + value: 36.301 + - type: ndcg_at_1000 + value: 38.318999999999996 + - type: map_at_1 + value: 13.793 + - type: map_at_3 + value: 20.237 + - type: map_at_5 + value: 22.32 + - type: map_at_10 + value: 23.829 + - type: map_at_20 + value: 24.596999999999998 + - type: map_at_100 + value: 25.117 + - type: map_at_1000 + value: 25.194 + - type: recall_at_1 + value: 13.793 + - type: recall_at_3 + value: 28.592000000000002 + - type: recall_at_5 + value: 37.556 + - type: recall_at_10 + value: 48.669000000000004 + - type: recall_at_20 + value: 59.379000000000005 + - type: recall_at_100 + value: 78.927 + - type: recall_at_1000 + value: 94.568 + - type: precision_at_1 + value: 14.155000000000001 + - type: precision_at_3 + value: 9.828000000000001 + - type: precision_at_5 + value: 7.785 + - type: precision_at_10 + value: 5.06 + - type: precision_at_20 + value: 3.097 + - type: precision_at_100 + value: 0.83 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 14.1547 + - type: mrr_at_3 + value: 20.7139 + - type: mrr_at_5 + value: 22.8028 + - type: mrr_at_10 + value: 24.3047 + - type: mrr_at_20 + value: 25.0548 + - type: mrr_at_100 + value: 25.552000000000003 + - type: mrr_at_1000 + value: 25.622 + - type: nauc_ndcg_at_1_max + value: 1.4238 + - type: nauc_ndcg_at_1_std + value: -13.091800000000001 + - type: nauc_ndcg_at_1_diff1 + value: 29.1051 + - type: nauc_ndcg_at_3_max + value: 2.6131 + - type: nauc_ndcg_at_3_std + value: -14.6122 + - type: nauc_ndcg_at_3_diff1 + value: 24.0988 + - type: nauc_ndcg_at_5_max + value: 2.3456 + - type: nauc_ndcg_at_5_std + value: -15.092500000000001 + - type: nauc_ndcg_at_5_diff1 + value: 23.5516 + - type: nauc_ndcg_at_10_max + value: 2.8182 + - type: nauc_ndcg_at_10_std + value: -14.623700000000001 + - type: nauc_ndcg_at_10_diff1 + value: 23.1711 + - type: nauc_ndcg_at_20_max + value: 3.5518 + - type: nauc_ndcg_at_20_std + value: -12.931500000000002 + - type: nauc_ndcg_at_20_diff1 + value: 23.1818 + - type: nauc_ndcg_at_100_max + value: 4.7755 + - type: nauc_ndcg_at_100_std + value: -9.851899999999999 + - type: nauc_ndcg_at_100_diff1 + value: 23.340700000000002 + - type: nauc_ndcg_at_1000_max + value: 4.5916 + - type: nauc_ndcg_at_1000_std + value: -10.4923 + - type: nauc_ndcg_at_1000_diff1 + value: 23.5174 + - type: nauc_map_at_1_max + value: 1.4764 + - type: nauc_map_at_1_std + value: -13.2414 + - type: nauc_map_at_1_diff1 + value: 29.1169 + - type: nauc_map_at_3_max + value: 2.3523 + - type: nauc_map_at_3_std + value: -14.453 + - type: nauc_map_at_3_diff1 + value: 25.0786 + - type: nauc_map_at_5_max + value: 2.1924 + - type: nauc_map_at_5_std + value: -14.7681 + - type: nauc_map_at_5_diff1 + value: 24.7695 + - type: nauc_map_at_10_max + value: 2.3542 + - type: nauc_map_at_10_std + value: -14.6287 + - type: nauc_map_at_10_diff1 + value: 24.6169 + - type: nauc_map_at_20_max + value: 2.5815 + - type: nauc_map_at_20_std + value: -14.141699999999998 + - type: nauc_map_at_20_diff1 + value: 24.6406 + - type: nauc_map_at_100_max + value: 2.7435 + - type: nauc_map_at_100_std + value: -13.7208 + - type: nauc_map_at_100_diff1 + value: 24.6504 + - type: nauc_map_at_1000_max + value: 2.7392 + - type: nauc_map_at_1000_std + value: -13.7302 + - type: nauc_map_at_1000_diff1 + value: 24.654300000000003 + - type: nauc_recall_at_1_max + value: 1.4764 + - type: nauc_recall_at_1_std + value: -13.2414 + - type: nauc_recall_at_1_diff1 + value: 29.1169 + - type: nauc_recall_at_3_max + value: 3.2174 + - type: nauc_recall_at_3_std + value: -15.143300000000002 + - type: nauc_recall_at_3_diff1 + value: 21.593899999999998 + - type: nauc_recall_at_5_max + value: 2.6845 + - type: nauc_recall_at_5_std + value: -15.9795 + - type: nauc_recall_at_5_diff1 + value: 20.567 + - type: nauc_recall_at_10_max + value: 3.913 + - type: nauc_recall_at_10_std + value: -14.566899999999999 + - type: nauc_recall_at_10_diff1 + value: 19.4393 + - type: nauc_recall_at_20_max + value: 6.5038 + - type: nauc_recall_at_20_std + value: -8.572799999999999 + - type: nauc_recall_at_20_diff1 + value: 19.0899 + - type: nauc_recall_at_100_max + value: 16.7968 + - type: nauc_recall_at_100_std + value: 15.837200000000001 + - type: nauc_recall_at_100_diff1 + value: 18.3296 + - type: nauc_recall_at_1000_max + value: 39.6225 + - type: nauc_recall_at_1000_std + value: 53.9736 + - type: nauc_recall_at_1000_diff1 + value: 12.565499999999998 + - type: nauc_precision_at_1_max + value: 1.4238 + - type: nauc_precision_at_1_std + value: -13.091800000000001 + - type: nauc_precision_at_1_diff1 + value: 29.1051 + - type: nauc_precision_at_3_max + value: 3.3477 + - type: nauc_precision_at_3_std + value: -14.8784 + - type: nauc_precision_at_3_diff1 + value: 21.8029 + - type: nauc_precision_at_5_max + value: 2.8493 + - type: nauc_precision_at_5_std + value: -15.767000000000001 + - type: nauc_precision_at_5_diff1 + value: 20.5677 + - type: nauc_precision_at_10_max + value: 4.2772 + - type: nauc_precision_at_10_std + value: -14.0627 + - type: nauc_precision_at_10_diff1 + value: 19.1205 + - type: nauc_precision_at_20_max + value: 7.135800000000001 + - type: nauc_precision_at_20_std + value: -7.5076 + - type: nauc_precision_at_20_diff1 + value: 18.0149 + - type: nauc_precision_at_100_max + value: 16.791 + - type: nauc_precision_at_100_std + value: 16.2346 + - type: nauc_precision_at_100_diff1 + value: 13.9316 + - type: nauc_precision_at_1000_max + value: 20.7529 + - type: nauc_precision_at_1000_std + value: 27.4859 + - type: nauc_precision_at_1000_diff1 + value: 3.9303 + - type: nauc_mrr_at_1_max + value: 1.4238 + - type: nauc_mrr_at_1_std + value: -13.091800000000001 + - type: nauc_mrr_at_1_diff1 + value: 29.1051 + - type: nauc_mrr_at_3_max + value: 2.3397 + - type: nauc_mrr_at_3_std + value: -14.1544 + - type: nauc_mrr_at_3_diff1 + value: 25.208799999999997 + - type: nauc_mrr_at_5_max + value: 2.1534 + - type: nauc_mrr_at_5_std + value: -14.4094 + - type: nauc_mrr_at_5_diff1 + value: 24.8258 + - type: nauc_mrr_at_10_max + value: 2.4274 + - type: nauc_mrr_at_10_std + value: -14.2121 + - type: nauc_mrr_at_10_diff1 + value: 24.6847 + - type: nauc_mrr_at_20_max + value: 2.6235999999999997 + - type: nauc_mrr_at_20_std + value: -13.736400000000001 + - type: nauc_mrr_at_20_diff1 + value: 24.6859 + - type: nauc_mrr_at_100_max + value: 2.7653 + - type: nauc_mrr_at_100_std + value: -13.358600000000001 + - type: nauc_mrr_at_100_diff1 + value: 24.7238 + - type: nauc_mrr_at_1000_max + value: 2.7588999999999997 + - type: nauc_mrr_at_1000_std + value: -13.373199999999999 + - type: nauc_mrr_at_1000_diff1 + value: 24.7274 + - type: main_score + value: 29.866999999999997 + task: + type: Retrieval + - dataset: + config: en + name: MTEB MTOPDomainClassification (en) + revision: d80d48c1eb48d3562165c59d59d0034df9fff0bf + split: test + type: mteb/mtop_domain + metrics: + - type: accuracy + value: 89.89970000000001 + - type: f1 + value: 89.6705 + - type: f1_weighted + value: 89.8682 + - type: main_score + value: 89.89970000000001 + task: + type: Classification + - dataset: + config: en + name: MTEB MTOPIntentClassification (en) + revision: ae001d0e6b1228650b7bd1c2c65fb50ad11a8aba + split: test + type: mteb/mtop_intent + metrics: + - type: accuracy + value: 60.26899999999999 + - type: f1 + value: 40.8003 + - type: f1_weighted + value: 63.033899999999996 + - type: main_score + value: 60.26899999999999 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveIntentClassification (en) + revision: 4672e20407010da34463acc759c162ca9734bca6 + split: test + type: mteb/amazon_massive_intent + metrics: + - type: accuracy + value: 63.9509 + - type: f1 + value: 60.7828 + - type: f1_weighted + value: 62.8 + - type: main_score + value: 63.9509 + task: + type: Classification + - dataset: + config: en + name: MTEB MassiveScenarioClassification (en) + revision: fad2c6e8459f9e1c45d9315f4953d921437d70f8 + split: test + type: mteb/amazon_massive_scenario + metrics: + - type: accuracy + value: 70.928 + - type: f1 + value: 69.4755 + - type: f1_weighted + value: 70.6366 + - type: main_score + value: 70.928 + task: + type: Classification + - dataset: + config: default + name: MTEB MedrxivClusteringP2P (default) + revision: e7a26af6f3ae46b30dde8737f02c07b1505bcc73 + split: test + type: mteb/medrxiv-clustering-p2p + metrics: + - type: v_measure + value: 31.522 + - type: v_measure_std + value: 1.5528 + - type: main_score + value: 31.522 + task: + type: Clustering + - dataset: + config: default + name: MTEB MedrxivClusteringS2S (default) + revision: 35191c8c0dca72d8ff3efcd72aa802307d469663 + split: test + type: mteb/medrxiv-clustering-s2s + metrics: + - type: v_measure + value: 28.572599999999998 + - type: v_measure_std + value: 1.8154 + - type: main_score + value: 28.572599999999998 + task: + type: Clustering + - dataset: + config: default + name: MTEB MindSmallReranking (default) + revision: 59042f120c80e8afa9cdbb224f67076cec0fc9a7 + split: test + type: mteb/mind_small + metrics: + - type: map + value: 30.5381 + - type: mrr + value: 31.574099999999998 + - type: nAUC_map_max + value: -19.592000000000002 + - type: nAUC_map_std + value: -3.0272 + - type: nAUC_map_diff1 + value: 14.0537 + - type: nAUC_mrr_max + value: -13.974900000000002 + - type: nAUC_mrr_std + value: -0.8847 + - type: nAUC_mrr_diff1 + value: 13.2721 + - type: main_score + value: 30.5381 + task: + type: Reranking + - dataset: + config: default + name: MTEB NFCorpus (default) + revision: ec0fa4fe99da2ff19ca1214b7966684033a58814 + split: test + type: mteb/nfcorpus + metrics: + - type: ndcg_at_1 + value: 38.080000000000005 + - type: ndcg_at_3 + value: 34.405 + - type: ndcg_at_5 + value: 32.019999999999996 + - type: ndcg_at_10 + value: 28.903000000000002 + - type: ndcg_at_20 + value: 26.693 + - type: ndcg_at_100 + value: 26.662999999999997 + - type: ndcg_at_1000 + value: 35.698 + - type: map_at_1 + value: 4.423 + - type: map_at_3 + value: 7.733 + - type: map_at_5 + value: 9.006 + - type: map_at_10 + value: 10.366 + - type: map_at_20 + value: 11.333 + - type: map_at_100 + value: 12.811 + - type: map_at_1000 + value: 14.066 + - type: recall_at_1 + value: 4.423 + - type: recall_at_3 + value: 8.908000000000001 + - type: recall_at_5 + value: 11.179 + - type: recall_at_10 + value: 14.280999999999999 + - type: recall_at_20 + value: 17.192 + - type: recall_at_100 + value: 27.685 + - type: recall_at_1000 + value: 59.108000000000004 + - type: precision_at_1 + value: 40.248 + - type: precision_at_3 + value: 33.127 + - type: precision_at_5 + value: 27.864 + - type: precision_at_10 + value: 21.053 + - type: precision_at_20 + value: 15.356 + - type: precision_at_100 + value: 6.709 + - type: precision_at_1000 + value: 1.9529999999999998 + - type: mrr_at_1 + value: 40.247699999999995 + - type: mrr_at_3 + value: 47.7812 + - type: mrr_at_5 + value: 48.8958 + - type: mrr_at_10 + value: 49.4034 + - type: mrr_at_20 + value: 49.8468 + - type: mrr_at_100 + value: 50.104800000000004 + - type: mrr_at_1000 + value: 50.1703 + - type: nauc_ndcg_at_1_max + value: 34.5735 + - type: nauc_ndcg_at_1_std + value: 15.1084 + - type: nauc_ndcg_at_1_diff1 + value: 37.779 + - type: nauc_ndcg_at_3_max + value: 38.8071 + - type: nauc_ndcg_at_3_std + value: 24.7697 + - type: nauc_ndcg_at_3_diff1 + value: 29.5807 + - type: nauc_ndcg_at_5_max + value: 39.128800000000005 + - type: nauc_ndcg_at_5_std + value: 26.398 + - type: nauc_ndcg_at_5_diff1 + value: 30.3835 + - type: nauc_ndcg_at_10_max + value: 37.7665 + - type: nauc_ndcg_at_10_std + value: 27.5455 + - type: nauc_ndcg_at_10_diff1 + value: 30.1575 + - type: nauc_ndcg_at_20_max + value: 36.3537 + - type: nauc_ndcg_at_20_std + value: 28.4047 + - type: nauc_ndcg_at_20_diff1 + value: 27.9553 + - type: nauc_ndcg_at_100_max + value: 39.0086 + - type: nauc_ndcg_at_100_std + value: 28.4221 + - type: nauc_ndcg_at_100_diff1 + value: 27.833799999999997 + - type: nauc_ndcg_at_1000_max + value: 44.7295 + - type: nauc_ndcg_at_1000_std + value: 35.369 + - type: nauc_ndcg_at_1000_diff1 + value: 29.4449 + - type: nauc_map_at_1_max + value: 12.645100000000001 + - type: nauc_map_at_1_std + value: -13.536999999999999 + - type: nauc_map_at_1_diff1 + value: 45.0881 + - type: nauc_map_at_3_max + value: 14.6862 + - type: nauc_map_at_3_std + value: -6.6259 + - type: nauc_map_at_3_diff1 + value: 34.2575 + - type: nauc_map_at_5_max + value: 18.6559 + - type: nauc_map_at_5_std + value: -2.8853 + - type: nauc_map_at_5_diff1 + value: 32.9187 + - type: nauc_map_at_10_max + value: 22.1906 + - type: nauc_map_at_10_std + value: 1.8654 + - type: nauc_map_at_10_diff1 + value: 31.3784 + - type: nauc_map_at_20_max + value: 24.696199999999997 + - type: nauc_map_at_20_std + value: 6.1949 + - type: nauc_map_at_20_diff1 + value: 30.9956 + - type: nauc_map_at_100_max + value: 27.2011 + - type: nauc_map_at_100_std + value: 12.3619 + - type: nauc_map_at_100_diff1 + value: 30.811500000000002 + - type: nauc_map_at_1000_max + value: 27.6972 + - type: nauc_map_at_1000_std + value: 15.845999999999998 + - type: nauc_map_at_1000_diff1 + value: 30.5315 + - type: nauc_recall_at_1_max + value: 12.645100000000001 + - type: nauc_recall_at_1_std + value: -13.536999999999999 + - type: nauc_recall_at_1_diff1 + value: 45.0881 + - type: nauc_recall_at_3_max + value: 14.2305 + - type: nauc_recall_at_3_std + value: -2.4143000000000003 + - type: nauc_recall_at_3_diff1 + value: 27.1661 + - type: nauc_recall_at_5_max + value: 20.62 + - type: nauc_recall_at_5_std + value: 3.1332 + - type: nauc_recall_at_5_diff1 + value: 26.7813 + - type: nauc_recall_at_10_max + value: 22.0278 + - type: nauc_recall_at_10_std + value: 4.587 + - type: nauc_recall_at_10_diff1 + value: 22.0275 + - type: nauc_recall_at_20_max + value: 23.4161 + - type: nauc_recall_at_20_std + value: 8.2901 + - type: nauc_recall_at_20_diff1 + value: 20.9799 + - type: nauc_recall_at_100_max + value: 24.5345 + - type: nauc_recall_at_100_std + value: 17.1618 + - type: nauc_recall_at_100_diff1 + value: 15.586500000000001 + - type: nauc_recall_at_1000_max + value: 22.3168 + - type: nauc_recall_at_1000_std + value: 22.6961 + - type: nauc_recall_at_1000_diff1 + value: 9.9602 + - type: nauc_precision_at_1_max + value: 36.549 + - type: nauc_precision_at_1_std + value: 16.6789 + - type: nauc_precision_at_1_diff1 + value: 35.6095 + - type: nauc_precision_at_3_max + value: 42.6539 + - type: nauc_precision_at_3_std + value: 33.0974 + - type: nauc_precision_at_3_diff1 + value: 21.9208 + - type: nauc_precision_at_5_max + value: 41.787800000000004 + - type: nauc_precision_at_5_std + value: 35.2286 + - type: nauc_precision_at_5_diff1 + value: 21.104899999999997 + - type: nauc_precision_at_10_max + value: 37.7473 + - type: nauc_precision_at_10_std + value: 39.887 + - type: nauc_precision_at_10_diff1 + value: 18.9082 + - type: nauc_precision_at_20_max + value: 32.0874 + - type: nauc_precision_at_20_std + value: 44.798100000000005 + - type: nauc_precision_at_20_diff1 + value: 12.953000000000001 + - type: nauc_precision_at_100_max + value: 19.108900000000002 + - type: nauc_precision_at_100_std + value: 44.49 + - type: nauc_precision_at_100_diff1 + value: 6.4374 + - type: nauc_precision_at_1000_max + value: 2.5292 + - type: nauc_precision_at_1000_std + value: 30.523400000000002 + - type: nauc_precision_at_1000_diff1 + value: -0.6787 + - type: nauc_mrr_at_1_max + value: 36.549 + - type: nauc_mrr_at_1_std + value: 16.6789 + - type: nauc_mrr_at_1_diff1 + value: 35.6095 + - type: nauc_mrr_at_3_max + value: 43.425599999999996 + - type: nauc_mrr_at_3_std + value: 28.8242 + - type: nauc_mrr_at_3_diff1 + value: 33.4411 + - type: nauc_mrr_at_5_max + value: 44.5717 + - type: nauc_mrr_at_5_std + value: 29.5765 + - type: nauc_mrr_at_5_diff1 + value: 34.463899999999995 + - type: nauc_mrr_at_10_max + value: 44.6062 + - type: nauc_mrr_at_10_std + value: 29.5773 + - type: nauc_mrr_at_10_diff1 + value: 34.5158 + - type: nauc_mrr_at_20_max + value: 44.6961 + - type: nauc_mrr_at_20_std + value: 29.5126 + - type: nauc_mrr_at_20_diff1 + value: 34.2436 + - type: nauc_mrr_at_100_max + value: 44.8207 + - type: nauc_mrr_at_100_std + value: 29.649700000000003 + - type: nauc_mrr_at_100_diff1 + value: 34.3576 + - type: nauc_mrr_at_1000_max + value: 44.7763 + - type: nauc_mrr_at_1000_std + value: 29.6044 + - type: nauc_mrr_at_1000_diff1 + value: 34.3718 + - type: main_score + value: 28.903000000000002 + task: + type: Retrieval + - dataset: + config: default + name: MTEB NQ (default) + revision: b774495ed302d8c44a3a7ea25c90dbce03968f31 + split: test + type: mteb/nq + metrics: + - type: ndcg_at_1 + value: 34.589 + - type: ndcg_at_3 + value: 45.289 + - type: ndcg_at_5 + value: 49.919000000000004 + - type: ndcg_at_10 + value: 53.410000000000004 + - type: ndcg_at_20 + value: 55.786 + - type: ndcg_at_100 + value: 57.75599999999999 + - type: ndcg_at_1000 + value: 58.51499999999999 + - type: map_at_1 + value: 30.503999999999998 + - type: map_at_3 + value: 41.396 + - type: map_at_5 + value: 44.216 + - type: map_at_10 + value: 45.802 + - type: map_at_20 + value: 46.542 + - type: map_at_100 + value: 46.867999999999995 + - type: map_at_1000 + value: 46.903 + - type: recall_at_1 + value: 30.503999999999998 + - type: recall_at_3 + value: 53.244 + - type: recall_at_5 + value: 63.912 + - type: recall_at_10 + value: 74.06099999999999 + - type: recall_at_20 + value: 82.819 + - type: recall_at_100 + value: 92.51599999999999 + - type: recall_at_1000 + value: 98.156 + - type: precision_at_1 + value: 34.589 + - type: precision_at_3 + value: 20.693 + - type: precision_at_5 + value: 15.058 + - type: precision_at_10 + value: 8.818 + - type: precision_at_20 + value: 4.9799999999999995 + - type: precision_at_100 + value: 1.125 + - type: precision_at_1000 + value: 0.11900000000000001 + - type: mrr_at_1 + value: 34.617599999999996 + - type: mrr_at_3 + value: 44.7277 + - type: mrr_at_5 + value: 47.0408 + - type: mrr_at_10 + value: 48.335499999999996 + - type: mrr_at_20 + value: 48.8925 + - type: mrr_at_100 + value: 49.1307 + - type: mrr_at_1000 + value: 49.154199999999996 + - type: nauc_ndcg_at_1_max + value: 23.8893 + - type: nauc_ndcg_at_1_std + value: -3.0092 + - type: nauc_ndcg_at_1_diff1 + value: 36.789899999999996 + - type: nauc_ndcg_at_3_max + value: 26.161800000000003 + - type: nauc_ndcg_at_3_std + value: -3.6557 + - type: nauc_ndcg_at_3_diff1 + value: 31.381500000000003 + - type: nauc_ndcg_at_5_max + value: 28.4273 + - type: nauc_ndcg_at_5_std + value: -2.6271 + - type: nauc_ndcg_at_5_diff1 + value: 30.960700000000003 + - type: nauc_ndcg_at_10_max + value: 29.1744 + - type: nauc_ndcg_at_10_std + value: -0.9882 + - type: nauc_ndcg_at_10_diff1 + value: 30.9664 + - type: nauc_ndcg_at_20_max + value: 30.1188 + - type: nauc_ndcg_at_20_std + value: 0.6556000000000001 + - type: nauc_ndcg_at_20_diff1 + value: 30.8734 + - type: nauc_ndcg_at_100_max + value: 29.822 + - type: nauc_ndcg_at_100_std + value: 1.1388 + - type: nauc_ndcg_at_100_diff1 + value: 31.348300000000002 + - type: nauc_ndcg_at_1000_max + value: 29.1591 + - type: nauc_ndcg_at_1000_std + value: 0.22569999999999998 + - type: nauc_ndcg_at_1000_diff1 + value: 31.7286 + - type: nauc_map_at_1_max + value: 22.2587 + - type: nauc_map_at_1_std + value: -4.6109 + - type: nauc_map_at_1_diff1 + value: 37.0942 + - type: nauc_map_at_3_max + value: 25.3764 + - type: nauc_map_at_3_std + value: -4.1876 + - type: nauc_map_at_3_diff1 + value: 32.752700000000004 + - type: nauc_map_at_5_max + value: 26.6367 + - type: nauc_map_at_5_std + value: -3.6224 + - type: nauc_map_at_5_diff1 + value: 32.4957 + - type: nauc_map_at_10_max + value: 27.0304 + - type: nauc_map_at_10_std + value: -2.852 + - type: nauc_map_at_10_diff1 + value: 32.548899999999996 + - type: nauc_map_at_20_max + value: 27.2991 + - type: nauc_map_at_20_std + value: -2.3765 + - type: nauc_map_at_20_diff1 + value: 32.5216 + - type: nauc_map_at_100_max + value: 27.2665 + - type: nauc_map_at_100_std + value: -2.2849999999999997 + - type: nauc_map_at_100_diff1 + value: 32.5791 + - type: nauc_map_at_1000_max + value: 27.243499999999997 + - type: nauc_map_at_1000_std + value: -2.3154999999999997 + - type: nauc_map_at_1000_diff1 + value: 32.5925 + - type: nauc_recall_at_1_max + value: 22.2587 + - type: nauc_recall_at_1_std + value: -4.6109 + - type: nauc_recall_at_1_diff1 + value: 37.0942 + - type: nauc_recall_at_3_max + value: 27.0818 + - type: nauc_recall_at_3_std + value: -3.5904 + - type: nauc_recall_at_3_diff1 + value: 26.6279 + - type: nauc_recall_at_5_max + value: 32.6179 + - type: nauc_recall_at_5_std + value: -1.2186000000000001 + - type: nauc_recall_at_5_diff1 + value: 24.7151 + - type: nauc_recall_at_10_max + value: 36.105599999999995 + - type: nauc_recall_at_10_std + value: 4.5315 + - type: nauc_recall_at_10_diff1 + value: 23.4044 + - type: nauc_recall_at_20_max + value: 45.2605 + - type: nauc_recall_at_20_std + value: 17.092299999999998 + - type: nauc_recall_at_20_diff1 + value: 20.5304 + - type: nauc_recall_at_100_max + value: 57.85829999999999 + - type: nauc_recall_at_100_std + value: 42.517500000000005 + - type: nauc_recall_at_100_diff1 + value: 19.6591 + - type: nauc_recall_at_1000_max + value: 75.3601 + - type: nauc_recall_at_1000_std + value: 69.4265 + - type: nauc_recall_at_1000_diff1 + value: 29.8635 + - type: nauc_precision_at_1_max + value: 23.8893 + - type: nauc_precision_at_1_std + value: -3.0092 + - type: nauc_precision_at_1_diff1 + value: 36.789899999999996 + - type: nauc_precision_at_3_max + value: 27.1749 + - type: nauc_precision_at_3_std + value: -0.9776 + - type: nauc_precision_at_3_diff1 + value: 22.9551 + - type: nauc_precision_at_5_max + value: 28.6992 + - type: nauc_precision_at_5_std + value: 2.1732 + - type: nauc_precision_at_5_diff1 + value: 17.6422 + - type: nauc_precision_at_10_max + value: 27.2755 + - type: nauc_precision_at_10_std + value: 8.4934 + - type: nauc_precision_at_10_diff1 + value: 12.1581 + - type: nauc_precision_at_20_max + value: 26.858900000000002 + - type: nauc_precision_at_20_std + value: 15.7942 + - type: nauc_precision_at_20_diff1 + value: 5.8980999999999995 + - type: nauc_precision_at_100_max + value: 18.8392 + - type: nauc_precision_at_100_std + value: 19.7054 + - type: nauc_precision_at_100_diff1 + value: -0.8163 + - type: nauc_precision_at_1000_max + value: 9.8054 + - type: nauc_precision_at_1000_std + value: 14.4735 + - type: nauc_precision_at_1000_diff1 + value: -4.7447 + - type: nauc_mrr_at_1_max + value: 23.8759 + - type: nauc_mrr_at_1_std + value: -3.0908 + - type: nauc_mrr_at_1_diff1 + value: 36.7027 + - type: nauc_mrr_at_3_max + value: 25.9165 + - type: nauc_mrr_at_3_std + value: -2.3997 + - type: nauc_mrr_at_3_diff1 + value: 32.5473 + - type: nauc_mrr_at_5_max + value: 27.1119 + - type: nauc_mrr_at_5_std + value: -1.8426999999999998 + - type: nauc_mrr_at_5_diff1 + value: 32.4999 + - type: nauc_mrr_at_10_max + value: 27.2217 + - type: nauc_mrr_at_10_std + value: -1.3365 + - type: nauc_mrr_at_10_diff1 + value: 32.5293 + - type: nauc_mrr_at_20_max + value: 27.3157 + - type: nauc_mrr_at_20_std + value: -1.1132 + - type: nauc_mrr_at_20_diff1 + value: 32.554300000000005 + - type: nauc_mrr_at_100_max + value: 27.2621 + - type: nauc_mrr_at_100_std + value: -1.0897000000000001 + - type: nauc_mrr_at_100_diff1 + value: 32.6073 + - type: nauc_mrr_at_1000_max + value: 27.2409 + - type: nauc_mrr_at_1000_std + value: -1.1176 + - type: nauc_mrr_at_1000_diff1 + value: 32.6192 + - type: main_score + value: 53.410000000000004 + task: + type: Retrieval + - dataset: + config: default + name: MTEB QuoraRetrieval (default) + revision: e4e08e0b7dbe3c8700f0daef558ff32256715259 + split: test + type: mteb/quora + metrics: + - type: ndcg_at_1 + value: 79.64 + - type: ndcg_at_3 + value: 83.67599999999999 + - type: ndcg_at_5 + value: 85.52 + - type: ndcg_at_10 + value: 86.871 + - type: ndcg_at_20 + value: 87.59 + - type: ndcg_at_100 + value: 88.211 + - type: ndcg_at_1000 + value: 88.36 + - type: map_at_1 + value: 69.133 + - type: map_at_3 + value: 79.776 + - type: map_at_5 + value: 81.747 + - type: map_at_10 + value: 82.852 + - type: map_at_20 + value: 83.282 + - type: map_at_100 + value: 83.5 + - type: map_at_1000 + value: 83.519 + - type: recall_at_1 + value: 69.133 + - type: recall_at_3 + value: 85.526 + - type: recall_at_5 + value: 90.596 + - type: recall_at_10 + value: 94.613 + - type: recall_at_20 + value: 96.92699999999999 + - type: recall_at_100 + value: 99.24300000000001 + - type: recall_at_1000 + value: 99.96000000000001 + - type: precision_at_1 + value: 79.64 + - type: precision_at_3 + value: 36.516999999999996 + - type: precision_at_5 + value: 24.194 + - type: precision_at_10 + value: 13.203000000000001 + - type: precision_at_20 + value: 7.02 + - type: precision_at_100 + value: 1.514 + - type: precision_at_1000 + value: 0.156 + - type: mrr_at_1 + value: 79.60000000000001 + - type: mrr_at_3 + value: 84.955 + - type: mrr_at_5 + value: 85.74000000000001 + - type: mrr_at_10 + value: 86.0913 + - type: mrr_at_20 + value: 86.1768 + - type: mrr_at_100 + value: 86.2076 + - type: mrr_at_1000 + value: 86.2092 + - type: nauc_ndcg_at_1_max + value: 39.4509 + - type: nauc_ndcg_at_1_std + value: -30.6309 + - type: nauc_ndcg_at_1_diff1 + value: 76.5171 + - type: nauc_ndcg_at_3_max + value: 37.9586 + - type: nauc_ndcg_at_3_std + value: -35.8174 + - type: nauc_ndcg_at_3_diff1 + value: 74.5992 + - type: nauc_ndcg_at_5_max + value: 38.541799999999995 + - type: nauc_ndcg_at_5_std + value: -36.456300000000006 + - type: nauc_ndcg_at_5_diff1 + value: 75.0506 + - type: nauc_ndcg_at_10_max + value: 38.996199999999995 + - type: nauc_ndcg_at_10_std + value: -35.6649 + - type: nauc_ndcg_at_10_diff1 + value: 75.3601 + - type: nauc_ndcg_at_20_max + value: 39.1758 + - type: nauc_ndcg_at_20_std + value: -34.7636 + - type: nauc_ndcg_at_20_diff1 + value: 75.3846 + - type: nauc_ndcg_at_100_max + value: 39.6116 + - type: nauc_ndcg_at_100_std + value: -33.2361 + - type: nauc_ndcg_at_100_diff1 + value: 75.31 + - type: nauc_ndcg_at_1000_max + value: 39.6171 + - type: nauc_ndcg_at_1000_std + value: -33.1588 + - type: nauc_ndcg_at_1000_diff1 + value: 75.2929 + - type: nauc_map_at_1_max + value: 28.8061 + - type: nauc_map_at_1_std + value: -33.7016 + - type: nauc_map_at_1_diff1 + value: 78.7612 + - type: nauc_map_at_3_max + value: 35.2541 + - type: nauc_map_at_3_std + value: -37.741400000000006 + - type: nauc_map_at_3_diff1 + value: 75.8173 + - type: nauc_map_at_5_max + value: 36.822500000000005 + - type: nauc_map_at_5_std + value: -37.710300000000004 + - type: nauc_map_at_5_diff1 + value: 75.7355 + - type: nauc_map_at_10_max + value: 37.5769 + - type: nauc_map_at_10_std + value: -36.5907 + - type: nauc_map_at_10_diff1 + value: 75.60040000000001 + - type: nauc_map_at_20_max + value: 37.8409 + - type: nauc_map_at_20_std + value: -35.7977 + - type: nauc_map_at_20_diff1 + value: 75.4885 + - type: nauc_map_at_100_max + value: 38.0097 + - type: nauc_map_at_100_std + value: -35.1815 + - type: nauc_map_at_100_diff1 + value: 75.4349 + - type: nauc_map_at_1000_max + value: 38.0191 + - type: nauc_map_at_1000_std + value: -35.1434 + - type: nauc_map_at_1000_diff1 + value: 75.4325 + - type: nauc_recall_at_1_max + value: 28.8061 + - type: nauc_recall_at_1_std + value: -33.7016 + - type: nauc_recall_at_1_diff1 + value: 78.7612 + - type: nauc_recall_at_3_max + value: 32.889 + - type: nauc_recall_at_3_std + value: -41.323100000000004 + - type: nauc_recall_at_3_diff1 + value: 71.73570000000001 + - type: nauc_recall_at_5_max + value: 34.6917 + - type: nauc_recall_at_5_std + value: -44.5216 + - type: nauc_recall_at_5_diff1 + value: 70.42540000000001 + - type: nauc_recall_at_10_max + value: 36.0356 + - type: nauc_recall_at_10_std + value: -45.073 + - type: nauc_recall_at_10_diff1 + value: 70.1776 + - type: nauc_recall_at_20_max + value: 35.714800000000004 + - type: nauc_recall_at_20_std + value: -44.0962 + - type: nauc_recall_at_20_diff1 + value: 71.23620000000001 + - type: nauc_recall_at_100_max + value: 43.105199999999996 + - type: nauc_recall_at_100_std + value: -18.800900000000002 + - type: nauc_recall_at_100_diff1 + value: 70.7888 + - type: nauc_recall_at_1000_max + value: 64.4844 + - type: nauc_recall_at_1000_std + value: 41.486200000000004 + - type: nauc_recall_at_1000_diff1 + value: 69.0643 + - type: nauc_precision_at_1_max + value: 39.4509 + - type: nauc_precision_at_1_std + value: -30.6309 + - type: nauc_precision_at_1_diff1 + value: 76.5171 + - type: nauc_precision_at_3_max + value: 12.514800000000001 + - type: nauc_precision_at_3_std + value: 3.2272000000000003 + - type: nauc_precision_at_3_diff1 + value: -11.8298 + - type: nauc_precision_at_5_max + value: 6.0901 + - type: nauc_precision_at_5_std + value: 12.6778 + - type: nauc_precision_at_5_diff1 + value: -26.570300000000003 + - type: nauc_precision_at_10_max + value: 0.9773999999999999 + - type: nauc_precision_at_10_std + value: 21.1764 + - type: nauc_precision_at_10_diff1 + value: -35.2909 + - type: nauc_precision_at_20_max + value: -2.2387 + - type: nauc_precision_at_20_std + value: 26.571099999999998 + - type: nauc_precision_at_20_diff1 + value: -39.0582 + - type: nauc_precision_at_100_max + value: -4.9125000000000005 + - type: nauc_precision_at_100_std + value: 31.9907 + - type: nauc_precision_at_100_diff1 + value: -41.5916 + - type: nauc_precision_at_1000_max + value: -6.0841 + - type: nauc_precision_at_1000_std + value: 32.8504 + - type: nauc_precision_at_1000_diff1 + value: -42.25 + - type: nauc_mrr_at_1_max + value: 39.285599999999995 + - type: nauc_mrr_at_1_std + value: -30.799100000000003 + - type: nauc_mrr_at_1_diff1 + value: 76.6113 + - type: nauc_mrr_at_3_max + value: 40.7492 + - type: nauc_mrr_at_3_std + value: -31.933699999999998 + - type: nauc_mrr_at_3_diff1 + value: 75.593 + - type: nauc_mrr_at_5_max + value: 40.87 + - type: nauc_mrr_at_5_std + value: -31.9333 + - type: nauc_mrr_at_5_diff1 + value: 75.7331 + - type: nauc_mrr_at_10_max + value: 40.7704 + - type: nauc_mrr_at_10_std + value: -31.839699999999997 + - type: nauc_mrr_at_10_diff1 + value: 75.8249 + - type: nauc_mrr_at_20_max + value: 40.7107 + - type: nauc_mrr_at_20_std + value: -31.7701 + - type: nauc_mrr_at_20_diff1 + value: 75.8463 + - type: nauc_mrr_at_100_max + value: 40.6937 + - type: nauc_mrr_at_100_std + value: -31.735999999999997 + - type: nauc_mrr_at_100_diff1 + value: 75.84309999999999 + - type: nauc_mrr_at_1000_max + value: 40.691 + - type: nauc_mrr_at_1000_std + value: -31.7368 + - type: nauc_mrr_at_1000_diff1 + value: 75.84349999999999 + - type: main_score + value: 86.871 + task: + type: Retrieval + - dataset: + config: default + name: MTEB RedditClustering (default) + revision: 24640382cdbf8abc73003fb0fa6d111a705499eb + split: test + type: mteb/reddit-clustering + metrics: + - type: v_measure + value: 45.8568 + - type: v_measure_std + value: 5.685 + - type: main_score + value: 45.8568 + task: + type: Clustering + - dataset: + config: default + name: MTEB RedditClusteringP2P (default) + revision: 385e3cb46b4cfa89021f56c4380204149d0efe33 + split: test + type: mteb/reddit-clustering-p2p + metrics: + - type: v_measure + value: 54.9896 + - type: v_measure_std + value: 12.0517 + - type: main_score + value: 54.9896 + task: + type: Clustering + - dataset: + config: default + name: MTEB SCIDOCS (default) + revision: f8c2fcf00f625baaa80f62ec5bd9e1fff3b8ae88 + split: test + type: mteb/scidocs + metrics: + - type: ndcg_at_1 + value: 20.599999999999998 + - type: ndcg_at_3 + value: 17.214 + - type: ndcg_at_5 + value: 14.93 + - type: ndcg_at_10 + value: 17.721 + - type: ndcg_at_20 + value: 20.619 + - type: ndcg_at_100 + value: 25.46 + - type: ndcg_at_1000 + value: 30.846 + - type: map_at_1 + value: 4.175 + - type: map_at_3 + value: 7.611 + - type: map_at_5 + value: 8.955 + - type: map_at_10 + value: 10.360999999999999 + - type: map_at_20 + value: 11.414 + - type: map_at_100 + value: 12.3 + - type: map_at_1000 + value: 12.595999999999998 + - type: recall_at_1 + value: 4.175 + - type: recall_at_3 + value: 9.868 + - type: recall_at_5 + value: 13.303 + - type: recall_at_10 + value: 18.397 + - type: recall_at_20 + value: 25.162000000000003 + - type: recall_at_100 + value: 40.99 + - type: recall_at_1000 + value: 67.322 + - type: precision_at_1 + value: 20.599999999999998 + - type: precision_at_3 + value: 16.2 + - type: precision_at_5 + value: 13.120000000000001 + - type: precision_at_10 + value: 9.06 + - type: precision_at_20 + value: 6.1899999999999995 + - type: precision_at_100 + value: 2.017 + - type: precision_at_1000 + value: 0.331 + - type: mrr_at_1 + value: 20.599999999999998 + - type: mrr_at_3 + value: 28.1833 + - type: mrr_at_5 + value: 30.043300000000002 + - type: mrr_at_10 + value: 31.1391 + - type: mrr_at_20 + value: 31.9095 + - type: mrr_at_100 + value: 32.3914 + - type: mrr_at_1000 + value: 32.4509 + - type: nauc_ndcg_at_1_max + value: 26.9024 + - type: nauc_ndcg_at_1_std + value: 4.1442 + - type: nauc_ndcg_at_1_diff1 + value: 25.9169 + - type: nauc_ndcg_at_3_max + value: 33.2338 + - type: nauc_ndcg_at_3_std + value: 7.0103 + - type: nauc_ndcg_at_3_diff1 + value: 24.8464 + - type: nauc_ndcg_at_5_max + value: 33.833999999999996 + - type: nauc_ndcg_at_5_std + value: 8.515 + - type: nauc_ndcg_at_5_diff1 + value: 22.7135 + - type: nauc_ndcg_at_10_max + value: 34.6873 + - type: nauc_ndcg_at_10_std + value: 12.3294 + - type: nauc_ndcg_at_10_diff1 + value: 20.4198 + - type: nauc_ndcg_at_20_max + value: 36.889 + - type: nauc_ndcg_at_20_std + value: 15.5519 + - type: nauc_ndcg_at_20_diff1 + value: 20.7428 + - type: nauc_ndcg_at_100_max + value: 39.0403 + - type: nauc_ndcg_at_100_std + value: 20.2488 + - type: nauc_ndcg_at_100_diff1 + value: 20.572 + - type: nauc_ndcg_at_1000_max + value: 38.7458 + - type: nauc_ndcg_at_1000_std + value: 21.7088 + - type: nauc_ndcg_at_1000_diff1 + value: 20.5603 + - type: nauc_map_at_1_max + value: 27.091199999999997 + - type: nauc_map_at_1_std + value: 4.3355999999999995 + - type: nauc_map_at_1_diff1 + value: 25.7587 + - type: nauc_map_at_3_max + value: 33.602900000000005 + - type: nauc_map_at_3_std + value: 5.8709 + - type: nauc_map_at_3_diff1 + value: 25.5351 + - type: nauc_map_at_5_max + value: 34.414 + - type: nauc_map_at_5_std + value: 6.914199999999999 + - type: nauc_map_at_5_diff1 + value: 23.7741 + - type: nauc_map_at_10_max + value: 35.1586 + - type: nauc_map_at_10_std + value: 10.078800000000001 + - type: nauc_map_at_10_diff1 + value: 21.628600000000002 + - type: nauc_map_at_20_max + value: 36.7719 + - type: nauc_map_at_20_std + value: 12.1807 + - type: nauc_map_at_20_diff1 + value: 22.0201 + - type: nauc_map_at_100_max + value: 37.5971 + - type: nauc_map_at_100_std + value: 13.828299999999999 + - type: nauc_map_at_100_diff1 + value: 21.8011 + - type: nauc_map_at_1000_max + value: 37.6524 + - type: nauc_map_at_1000_std + value: 14.0603 + - type: nauc_map_at_1000_diff1 + value: 21.87 + - type: nauc_recall_at_1_max + value: 27.091199999999997 + - type: nauc_recall_at_1_std + value: 4.3355999999999995 + - type: nauc_recall_at_1_diff1 + value: 25.7587 + - type: nauc_recall_at_3_max + value: 35.0346 + - type: nauc_recall_at_3_std + value: 7.6722 + - type: nauc_recall_at_3_diff1 + value: 23.8398 + - type: nauc_recall_at_5_max + value: 34.7429 + - type: nauc_recall_at_5_std + value: 9.8479 + - type: nauc_recall_at_5_diff1 + value: 19.9693 + - type: nauc_recall_at_10_max + value: 34.1188 + - type: nauc_recall_at_10_std + value: 16.0443 + - type: nauc_recall_at_10_diff1 + value: 14.844399999999998 + - type: nauc_recall_at_20_max + value: 36.9825 + - type: nauc_recall_at_20_std + value: 21.5553 + - type: nauc_recall_at_20_diff1 + value: 15.4056 + - type: nauc_recall_at_100_max + value: 37.238 + - type: nauc_recall_at_100_std + value: 30.425400000000003 + - type: nauc_recall_at_100_diff1 + value: 12.839 + - type: nauc_recall_at_1000_max + value: 30.188599999999997 + - type: nauc_recall_at_1000_std + value: 34.7768 + - type: nauc_recall_at_1000_diff1 + value: 8.337 + - type: nauc_precision_at_1_max + value: 26.9024 + - type: nauc_precision_at_1_std + value: 4.1442 + - type: nauc_precision_at_1_diff1 + value: 25.9169 + - type: nauc_precision_at_3_max + value: 35.3949 + - type: nauc_precision_at_3_std + value: 7.818300000000001 + - type: nauc_precision_at_3_diff1 + value: 24.4077 + - type: nauc_precision_at_5_max + value: 35.0653 + - type: nauc_precision_at_5_std + value: 10.1252 + - type: nauc_precision_at_5_diff1 + value: 20.4485 + - type: nauc_precision_at_10_max + value: 34.5799 + - type: nauc_precision_at_10_std + value: 16.2893 + - type: nauc_precision_at_10_diff1 + value: 15.337600000000002 + - type: nauc_precision_at_20_max + value: 37.47 + - type: nauc_precision_at_20_std + value: 21.7447 + - type: nauc_precision_at_20_diff1 + value: 15.644 + - type: nauc_precision_at_100_max + value: 37.8956 + - type: nauc_precision_at_100_std + value: 30.6388 + - type: nauc_precision_at_100_diff1 + value: 13.5011 + - type: nauc_precision_at_1000_max + value: 30.456699999999998 + - type: nauc_precision_at_1000_std + value: 34.3528 + - type: nauc_precision_at_1000_diff1 + value: 8.963899999999999 + - type: nauc_mrr_at_1_max + value: 26.9024 + - type: nauc_mrr_at_1_std + value: 4.1442 + - type: nauc_mrr_at_1_diff1 + value: 25.9169 + - type: nauc_mrr_at_3_max + value: 30.214999999999996 + - type: nauc_mrr_at_3_std + value: 7.4483 + - type: nauc_mrr_at_3_diff1 + value: 23.7169 + - type: nauc_mrr_at_5_max + value: 30.1892 + - type: nauc_mrr_at_5_std + value: 8.319 + - type: nauc_mrr_at_5_diff1 + value: 23.4187 + - type: nauc_mrr_at_10_max + value: 30.5879 + - type: nauc_mrr_at_10_std + value: 8.9701 + - type: nauc_mrr_at_10_diff1 + value: 23.4357 + - type: nauc_mrr_at_20_max + value: 30.579800000000002 + - type: nauc_mrr_at_20_std + value: 9.3186 + - type: nauc_mrr_at_20_diff1 + value: 23.2358 + - type: nauc_mrr_at_100_max + value: 30.660500000000003 + - type: nauc_mrr_at_100_std + value: 9.404 + - type: nauc_mrr_at_100_diff1 + value: 23.3937 + - type: nauc_mrr_at_1000_max + value: 30.6315 + - type: nauc_mrr_at_1000_std + value: 9.363299999999999 + - type: nauc_mrr_at_1000_diff1 + value: 23.392599999999998 + - type: main_score + value: 17.721 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SICK-R (default) + revision: 20a6d6f312dd54037fe07a32d58e5e168867909d + split: test + type: mteb/sickr-sts + metrics: + - type: pearson + value: 75.5378 + - type: spearman + value: 68.7448 + - type: cosine_pearson + value: 75.5378 + - type: cosine_spearman + value: 68.7448 + - type: manhattan_pearson + value: 72.905 + - type: manhattan_spearman + value: 68.9036 + - type: euclidean_pearson + value: 72.7586 + - type: euclidean_spearman + value: 68.7448 + - type: main_score + value: 68.7448 + task: + type: STS + - dataset: + config: default + name: MTEB STS12 (default) + revision: a0d554a64d88156834ff5ae9920b964011b16384 + split: test + type: mteb/sts12-sts + metrics: + - type: pearson + value: 81.6341 + - type: spearman + value: 75.1911 + - type: cosine_pearson + value: 81.6341 + - type: cosine_spearman + value: 75.1911 + - type: manhattan_pearson + value: 78.4046 + - type: manhattan_spearman + value: 75.1706 + - type: euclidean_pearson + value: 78.3649 + - type: euclidean_spearman + value: 75.1934 + - type: main_score + value: 75.1911 + task: + type: STS + - dataset: + config: default + name: MTEB STS13 (default) + revision: 7e90230a92c190f1bf69ae9002b8cea547a64cca + split: test + type: mteb/sts13-sts + metrics: + - type: pearson + value: 76.4378 + - type: spearman + value: 77.3053 + - type: cosine_pearson + value: 76.4378 + - type: cosine_spearman + value: 77.3053 + - type: manhattan_pearson + value: 77.1958 + - type: manhattan_spearman + value: 77.2543 + - type: euclidean_pearson + value: 77.2317 + - type: euclidean_spearman + value: 77.3053 + - type: main_score + value: 77.3053 + task: + type: STS + - dataset: + config: default + name: MTEB STS14 (default) + revision: 6031580fec1f6af667f0bd2da0a551cf4f0b2375 + split: test + type: mteb/sts14-sts + metrics: + - type: pearson + value: 78.4342 + - type: spearman + value: 74.9479 + - type: cosine_pearson + value: 78.4342 + - type: cosine_spearman + value: 74.9479 + - type: manhattan_pearson + value: 77.12219999999999 + - type: manhattan_spearman + value: 74.924 + - type: euclidean_pearson + value: 77.14800000000001 + - type: euclidean_spearman + value: 74.94800000000001 + - type: main_score + value: 74.9479 + task: + type: STS + - dataset: + config: default + name: MTEB STS15 (default) + revision: ae752c7c21bf194d8b67fd573edf7ae58183cbe3 + split: test + type: mteb/sts15-sts + metrics: + - type: pearson + value: 85.1908 + - type: spearman + value: 86.0174 + - type: cosine_pearson + value: 85.1908 + - type: cosine_spearman + value: 86.0174 + - type: manhattan_pearson + value: 85.4436 + - type: manhattan_spearman + value: 86.0332 + - type: euclidean_pearson + value: 85.4339 + - type: euclidean_spearman + value: 86.0174 + - type: main_score + value: 86.0174 + task: + type: STS + - dataset: + config: default + name: MTEB STS16 (default) + revision: 4d8694f8f0e0100860b497b999b3dbed754a0513 + split: test + type: mteb/sts16-sts + metrics: + - type: pearson + value: 80.5421 + - type: spearman + value: 81.9568 + - type: cosine_pearson + value: 80.5421 + - type: cosine_spearman + value: 81.9568 + - type: manhattan_pearson + value: 81.1013 + - type: manhattan_spearman + value: 81.8165 + - type: euclidean_pearson + value: 81.24510000000001 + - type: euclidean_spearman + value: 81.9568 + - type: main_score + value: 81.9568 + task: + type: STS + - dataset: + config: en-tr + name: MTEB STS17 (en-tr) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 48.2717 + - type: spearman + value: 44.642900000000004 + - type: cosine_pearson + value: 48.2717 + - type: cosine_spearman + value: 44.642900000000004 + - type: manhattan_pearson + value: 50.314400000000006 + - type: manhattan_spearman + value: 44.982299999999995 + - type: euclidean_pearson + value: 50.1685 + - type: euclidean_spearman + value: 44.642900000000004 + - type: main_score + value: 44.642900000000004 + task: + type: STS + - dataset: + config: it-en + name: MTEB STS17 (it-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 67.8601 + - type: spearman + value: 68.2763 + - type: cosine_pearson + value: 67.8601 + - type: cosine_spearman + value: 68.2763 + - type: manhattan_pearson + value: 68.1563 + - type: manhattan_spearman + value: 68.4724 + - type: euclidean_pearson + value: 68.1026 + - type: euclidean_spearman + value: 68.2763 + - type: main_score + value: 68.2763 + task: + type: STS + - dataset: + config: en-en + name: MTEB STS17 (en-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 78.05539999999999 + - type: spearman + value: 78.5929 + - type: cosine_pearson + value: 78.05539999999999 + - type: cosine_spearman + value: 78.5929 + - type: manhattan_pearson + value: 78.408 + - type: manhattan_spearman + value: 78.8622 + - type: euclidean_pearson + value: 78.1413 + - type: euclidean_spearman + value: 78.5929 + - type: main_score + value: 78.5929 + task: + type: STS + - dataset: + config: en-ar + name: MTEB STS17 (en-ar) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 59.4349 + - type: spearman + value: 59.838800000000006 + - type: cosine_pearson + value: 59.4349 + - type: cosine_spearman + value: 59.838800000000006 + - type: manhattan_pearson + value: 60.7565 + - type: manhattan_spearman + value: 60.5824 + - type: euclidean_pearson + value: 60.247099999999996 + - type: euclidean_spearman + value: 59.838800000000006 + - type: main_score + value: 59.838800000000006 + task: + type: STS + - dataset: + config: fr-en + name: MTEB STS17 (fr-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 73.84039999999999 + - type: spearman + value: 74.2498 + - type: cosine_pearson + value: 73.84039999999999 + - type: cosine_spearman + value: 74.2498 + - type: manhattan_pearson + value: 74.6784 + - type: manhattan_spearman + value: 74.4608 + - type: euclidean_pearson + value: 74.5596 + - type: euclidean_spearman + value: 74.2498 + - type: main_score + value: 74.2498 + task: + type: STS + - dataset: + config: nl-en + name: MTEB STS17 (nl-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 67.9218 + - type: spearman + value: 68.0418 + - type: cosine_pearson + value: 67.9218 + - type: cosine_spearman + value: 68.0418 + - type: manhattan_pearson + value: 68.51 + - type: manhattan_spearman + value: 68.1968 + - type: euclidean_pearson + value: 68.343 + - type: euclidean_spearman + value: 68.0418 + - type: main_score + value: 68.0418 + task: + type: STS + - dataset: + config: es-en + name: MTEB STS17 (es-en) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 70.381 + - type: spearman + value: 69.5729 + - type: cosine_pearson + value: 70.381 + - type: cosine_spearman + value: 69.5729 + - type: manhattan_pearson + value: 70.8688 + - type: manhattan_spearman + value: 69.4406 + - type: euclidean_pearson + value: 71.0267 + - type: euclidean_spearman + value: 69.5729 + - type: main_score + value: 69.5729 + task: + type: STS + - dataset: + config: en-de + name: MTEB STS17 (en-de) + revision: faeb762787bd10488a50c8b5be4a3b82e411949c + split: test + type: mteb/sts17-crosslingual-sts + metrics: + - type: pearson + value: 70.0196 + - type: spearman + value: 69.7175 + - type: cosine_pearson + value: 70.0196 + - type: cosine_spearman + value: 69.7175 + - type: manhattan_pearson + value: 71.40990000000001 + - type: manhattan_spearman + value: 70.1461 + - type: euclidean_pearson + value: 70.88799999999999 + - type: euclidean_spearman + value: 69.7175 + - type: main_score + value: 69.7175 + task: + type: STS + - dataset: + config: de-en + name: MTEB STS22 (de-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 65.7536 + - type: spearman + value: 60.04429999999999 + - type: cosine_pearson + value: 65.7536 + - type: cosine_spearman + value: 60.04429999999999 + - type: manhattan_pearson + value: 68.58579999999999 + - type: manhattan_spearman + value: 60.3699 + - type: euclidean_pearson + value: 68.3761 + - type: euclidean_spearman + value: 60.04429999999999 + - type: main_score + value: 60.04429999999999 + task: + type: STS + - dataset: + config: en + name: MTEB STS22 (en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 68.997 + - type: spearman + value: 68.1508 + - type: cosine_pearson + value: 68.997 + - type: cosine_spearman + value: 68.1508 + - type: manhattan_pearson + value: 68.9229 + - type: manhattan_spearman + value: 68.0124 + - type: euclidean_pearson + value: 69.0519 + - type: euclidean_spearman + value: 68.1508 + - type: main_score + value: 68.1508 + task: + type: STS + - dataset: + config: es-en + name: MTEB STS22 (es-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 80.2006 + - type: spearman + value: 80.4702 + - type: cosine_pearson + value: 80.2006 + - type: cosine_spearman + value: 80.4702 + - type: manhattan_pearson + value: 80.81009999999999 + - type: manhattan_spearman + value: 80.6037 + - type: euclidean_pearson + value: 80.66290000000001 + - type: euclidean_spearman + value: 80.4702 + - type: main_score + value: 80.4702 + task: + type: STS + - dataset: + config: zh-en + name: MTEB STS22 (zh-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 74.0885 + - type: spearman + value: 72.4574 + - type: cosine_pearson + value: 74.0885 + - type: cosine_spearman + value: 72.4574 + - type: manhattan_pearson + value: 75.25659999999999 + - type: manhattan_spearman + value: 71.9695 + - type: euclidean_pearson + value: 75.4999 + - type: euclidean_spearman + value: 72.4574 + - type: main_score + value: 72.4574 + task: + type: STS + - dataset: + config: pl-en + name: MTEB STS22 (pl-en) + revision: de9d86b3b84231dc21f76c7b7af1f28e2f57f6e3 + split: test + type: mteb/sts22-crosslingual-sts + metrics: + - type: pearson + value: 74.1794 + - type: spearman + value: 70.6749 + - type: cosine_pearson + value: 74.1794 + - type: cosine_spearman + value: 70.6749 + - type: manhattan_pearson + value: 74.3245 + - type: manhattan_spearman + value: 71.2375 + - type: euclidean_pearson + value: 73.221 + - type: euclidean_spearman + value: 70.6749 + - type: main_score + value: 70.6749 + task: + type: STS + - dataset: + config: default + name: MTEB STSBenchmark (default) + revision: b0fddb56ed78048fa8b90373c8a3cfc37b684831 + split: test + type: mteb/stsbenchmark-sts + metrics: + - type: pearson + value: 76.7328 + - type: spearman + value: 78.4076 + - type: cosine_pearson + value: 76.7328 + - type: cosine_spearman + value: 78.4076 + - type: manhattan_pearson + value: 78.24950000000001 + - type: manhattan_spearman + value: 78.23400000000001 + - type: euclidean_pearson + value: 78.3628 + - type: euclidean_spearman + value: 78.4076 + - type: main_score + value: 78.4076 + task: + type: STS + - dataset: + config: default + name: MTEB SciDocsRR (default) + revision: d3c5e1fc0b855ab6097bf1cda04dd73947d7caab + split: test + type: mteb/scidocs-reranking + metrics: + - type: map + value: 79.6097 + - type: mrr + value: 94.12939999999999 + - type: nAUC_map_max + value: 58.7937 + - type: nAUC_map_std + value: 69.6785 + - type: nAUC_map_diff1 + value: 7.4891 + - type: nAUC_mrr_max + value: 84.7821 + - type: nAUC_mrr_std + value: 77.6636 + - type: nAUC_mrr_diff1 + value: 49.763600000000004 + - type: main_score + value: 79.6097 + task: + type: Reranking + - dataset: + config: default + name: MTEB SciFact (default) + revision: 0228b52cf27578f30900b9e5271d331663a030d7 + split: test + type: mteb/scifact + metrics: + - type: ndcg_at_1 + value: 54.0 + - type: ndcg_at_3 + value: 60.851 + - type: ndcg_at_5 + value: 63.410999999999994 + - type: ndcg_at_10 + value: 65.847 + - type: ndcg_at_20 + value: 66.937 + - type: ndcg_at_100 + value: 68.262 + - type: ndcg_at_1000 + value: 69.341 + - type: map_at_1 + value: 51.093999999999994 + - type: map_at_3 + value: 58.044 + - type: map_at_5 + value: 59.702999999999996 + - type: map_at_10 + value: 60.885999999999996 + - type: map_at_20 + value: 61.266 + - type: map_at_100 + value: 61.482000000000006 + - type: map_at_1000 + value: 61.519 + - type: recall_at_1 + value: 51.093999999999994 + - type: recall_at_3 + value: 66.128 + - type: recall_at_5 + value: 72.456 + - type: recall_at_10 + value: 79.3 + - type: recall_at_20 + value: 83.2 + - type: recall_at_100 + value: 90.0 + - type: recall_at_1000 + value: 98.667 + - type: precision_at_1 + value: 54.0 + - type: precision_at_3 + value: 23.778 + - type: precision_at_5 + value: 15.933 + - type: precision_at_10 + value: 8.967 + - type: precision_at_20 + value: 4.75 + - type: precision_at_100 + value: 1.03 + - type: precision_at_1000 + value: 0.11199999999999999 + - type: mrr_at_1 + value: 54.0 + - type: mrr_at_3 + value: 60.3889 + - type: mrr_at_5 + value: 61.7556 + - type: mrr_at_10 + value: 62.5984 + - type: mrr_at_20 + value: 62.85039999999999 + - type: mrr_at_100 + value: 63.0155 + - type: mrr_at_1000 + value: 63.052699999999994 + - type: nauc_ndcg_at_1_max + value: 56.6373 + - type: nauc_ndcg_at_1_std + value: 2.1765 + - type: nauc_ndcg_at_1_diff1 + value: 71.14829999999999 + - type: nauc_ndcg_at_3_max + value: 53.7965 + - type: nauc_ndcg_at_3_std + value: -3.4057999999999997 + - type: nauc_ndcg_at_3_diff1 + value: 63.712199999999996 + - type: nauc_ndcg_at_5_max + value: 56.96059999999999 + - type: nauc_ndcg_at_5_std + value: 1.4794 + - type: nauc_ndcg_at_5_diff1 + value: 64.65419999999999 + - type: nauc_ndcg_at_10_max + value: 59.4154 + - type: nauc_ndcg_at_10_std + value: 5.2752 + - type: nauc_ndcg_at_10_diff1 + value: 64.3098 + - type: nauc_ndcg_at_20_max + value: 59.7717 + - type: nauc_ndcg_at_20_std + value: 6.2032 + - type: nauc_ndcg_at_20_diff1 + value: 64.18599999999999 + - type: nauc_ndcg_at_100_max + value: 59.2146 + - type: nauc_ndcg_at_100_std + value: 6.0138 + - type: nauc_ndcg_at_100_diff1 + value: 64.0895 + - type: nauc_ndcg_at_1000_max + value: 58.5714 + - type: nauc_ndcg_at_1000_std + value: 4.8872 + - type: nauc_ndcg_at_1000_diff1 + value: 64.66969999999999 + - type: nauc_map_at_1_max + value: 51.2417 + - type: nauc_map_at_1_std + value: -5.42 + - type: nauc_map_at_1_diff1 + value: 70.0616 + - type: nauc_map_at_3_max + value: 51.9587 + - type: nauc_map_at_3_std + value: -5.3035 + - type: nauc_map_at_3_diff1 + value: 65.282 + - type: nauc_map_at_5_max + value: 54.1516 + - type: nauc_map_at_5_std + value: -2.2858 + - type: nauc_map_at_5_diff1 + value: 65.86659999999999 + - type: nauc_map_at_10_max + value: 55.5412 + - type: nauc_map_at_10_std + value: -0.34299999999999997 + - type: nauc_map_at_10_diff1 + value: 65.89620000000001 + - type: nauc_map_at_20_max + value: 55.7967 + - type: nauc_map_at_20_std + value: 0.13799999999999998 + - type: nauc_map_at_20_diff1 + value: 65.8685 + - type: nauc_map_at_100_max + value: 55.74550000000001 + - type: nauc_map_at_100_std + value: 0.211 + - type: nauc_map_at_100_diff1 + value: 65.8557 + - type: nauc_map_at_1000_max + value: 55.728 + - type: nauc_map_at_1000_std + value: 0.1875 + - type: nauc_map_at_1000_diff1 + value: 65.8748 + - type: nauc_recall_at_1_max + value: 51.2417 + - type: nauc_recall_at_1_std + value: -5.42 + - type: nauc_recall_at_1_diff1 + value: 70.0616 + - type: nauc_recall_at_3_max + value: 52.4327 + - type: nauc_recall_at_3_std + value: -6.7153 + - type: nauc_recall_at_3_diff1 + value: 57.111999999999995 + - type: nauc_recall_at_5_max + value: 60.5827 + - type: nauc_recall_at_5_std + value: 7.1365 + - type: nauc_recall_at_5_diff1 + value: 58.3449 + - type: nauc_recall_at_10_max + value: 70.24770000000001 + - type: nauc_recall_at_10_std + value: 22.0896 + - type: nauc_recall_at_10_diff1 + value: 55.7264 + - type: nauc_recall_at_20_max + value: 73.483 + - type: nauc_recall_at_20_std + value: 29.653299999999998 + - type: nauc_recall_at_20_diff1 + value: 53.54750000000001 + - type: nauc_recall_at_100_max + value: 74.0321 + - type: nauc_recall_at_100_std + value: 37.491400000000006 + - type: nauc_recall_at_100_diff1 + value: 47.3918 + - type: nauc_recall_at_1000_max + value: 69.5378 + - type: nauc_recall_at_1000_std + value: 60.5042 + - type: nauc_recall_at_1000_diff1 + value: 19.5028 + - type: nauc_precision_at_1_max + value: 56.6373 + - type: nauc_precision_at_1_std + value: 2.1765 + - type: nauc_precision_at_1_diff1 + value: 71.14829999999999 + - type: nauc_precision_at_3_max + value: 51.811099999999996 + - type: nauc_precision_at_3_std + value: 8.4319 + - type: nauc_precision_at_3_diff1 + value: 48.545500000000004 + - type: nauc_precision_at_5_max + value: 55.4685 + - type: nauc_precision_at_5_std + value: 26.387 + - type: nauc_precision_at_5_diff1 + value: 39.6201 + - type: nauc_precision_at_10_max + value: 53.2436 + - type: nauc_precision_at_10_std + value: 41.6957 + - type: nauc_precision_at_10_diff1 + value: 24.6115 + - type: nauc_precision_at_20_max + value: 48.353699999999996 + - type: nauc_precision_at_20_std + value: 47.253 + - type: nauc_precision_at_20_diff1 + value: 15.687599999999998 + - type: nauc_precision_at_100_max + value: 36.771100000000004 + - type: nauc_precision_at_100_std + value: 48.1335 + - type: nauc_precision_at_100_diff1 + value: 2.6454 + - type: nauc_precision_at_1000_max + value: 23.0391 + - type: nauc_precision_at_1000_std + value: 53.26499999999999 + - type: nauc_precision_at_1000_diff1 + value: -15.0974 + - type: nauc_mrr_at_1_max + value: 56.6373 + - type: nauc_mrr_at_1_std + value: 2.1765 + - type: nauc_mrr_at_1_diff1 + value: 71.14829999999999 + - type: nauc_mrr_at_3_max + value: 57.6843 + - type: nauc_mrr_at_3_std + value: 2.4692 + - type: nauc_mrr_at_3_diff1 + value: 66.10340000000001 + - type: nauc_mrr_at_5_max + value: 59.2453 + - type: nauc_mrr_at_5_std + value: 5.1308 + - type: nauc_mrr_at_5_diff1 + value: 66.7377 + - type: nauc_mrr_at_10_max + value: 59.5575 + - type: nauc_mrr_at_10_std + value: 5.7778 + - type: nauc_mrr_at_10_diff1 + value: 66.36149999999999 + - type: nauc_mrr_at_20_max + value: 59.466300000000004 + - type: nauc_mrr_at_20_std + value: 5.6867 + - type: nauc_mrr_at_20_diff1 + value: 66.37100000000001 + - type: nauc_mrr_at_100_max + value: 59.404999999999994 + - type: nauc_mrr_at_100_std + value: 5.6528 + - type: nauc_mrr_at_100_diff1 + value: 66.41040000000001 + - type: nauc_mrr_at_1000_max + value: 59.3919 + - type: nauc_mrr_at_1000_std + value: 5.6358 + - type: nauc_mrr_at_1000_diff1 + value: 66.43050000000001 + - type: main_score + value: 65.847 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SprintDuplicateQuestions (default) + revision: d66bd1f72af766a5cc4b0ca5e00c162f89e8cc46 + split: test + type: mteb/sprintduplicatequestions-pairclassification + metrics: + - type: similarity_accuracy + value: 99.7386 + - type: similarity_accuracy_threshold + value: 84.1442 + - type: similarity_f1 + value: 86.41980000000001 + - type: similarity_f1_threshold + value: 84.1442 + - type: similarity_precision + value: 88.98310000000001 + - type: similarity_recall + value: 84.0 + - type: similarity_ap + value: 93.50309999999999 + - type: cosine_accuracy + value: 99.7386 + - type: cosine_accuracy_threshold + value: 84.1442 + - type: cosine_f1 + value: 86.41980000000001 + - type: cosine_f1_threshold + value: 84.1442 + - type: cosine_precision + value: 88.98310000000001 + - type: cosine_recall + value: 84.0 + - type: cosine_ap + value: 93.50309999999999 + - type: manhattan_accuracy + value: 99.7406 + - type: manhattan_accuracy_threshold + value: 1243.0971 + - type: manhattan_f1 + value: 86.5641 + - type: manhattan_f1_threshold + value: 1243.0971 + - type: manhattan_precision + value: 88.8421 + - type: manhattan_recall + value: 84.39999999999999 + - type: manhattan_ap + value: 93.50840000000001 + - type: euclidean_accuracy + value: 99.7386 + - type: euclidean_accuracy_threshold + value: 56.313 + - type: euclidean_f1 + value: 86.41980000000001 + - type: euclidean_f1_threshold + value: 56.313 + - type: euclidean_precision + value: 88.98310000000001 + - type: euclidean_recall + value: 84.0 + - type: euclidean_ap + value: 93.50309999999999 + - type: dot_accuracy + value: 99.7386 + - type: dot_accuracy_threshold + value: 84.1442 + - type: dot_f1 + value: 86.41980000000001 + - type: dot_f1_threshold + value: 84.1442 + - type: dot_precision + value: 88.98310000000001 + - type: dot_recall + value: 84.0 + - type: dot_ap + value: 93.50309999999999 + - type: max_accuracy + value: 99.7406 + - type: max_f1 + value: 86.5641 + - type: max_precision + value: 88.98310000000001 + - type: max_recall + value: 84.39999999999999 + - type: max_ap + value: 93.50840000000001 + - type: main_score + value: 93.50840000000001 + task: + type: PairClassification + - dataset: + config: default + name: MTEB StackExchangeClustering (default) + revision: 6cbc1f7b2bc0622f2e39d2c77fa502909748c259 + split: test + type: mteb/stackexchange-clustering + metrics: + - type: v_measure + value: 55.9311 + - type: v_measure_std + value: 5.0881 + - type: main_score + value: 55.9311 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackExchangeClusteringP2P (default) + revision: 815ca46b2622cec33ccafc3735d572c266efdb44 + split: test + type: mteb/stackexchange-clustering-p2p + metrics: + - type: v_measure + value: 32.9298 + - type: v_measure_std + value: 1.7169 + - type: main_score + value: 32.9298 + task: + type: Clustering + - dataset: + config: default + name: MTEB StackOverflowDupQuestions (default) + revision: e185fbe320c72810689fc5848eb6114e1ef5ec69 + split: test + type: mteb/stackoverflowdupquestions-reranking + metrics: + - type: map + value: 51.7759 + - type: mrr + value: 52.7456 + - type: nAUC_map_max + value: 15.138499999999999 + - type: nAUC_map_std + value: 9.876999999999999 + - type: nAUC_map_diff1 + value: 37.8337 + - type: nAUC_mrr_max + value: 16.128600000000002 + - type: nAUC_mrr_std + value: 10.4175 + - type: nAUC_mrr_diff1 + value: 37.3753 + - type: main_score + value: 51.7759 + task: + type: Reranking + - dataset: + config: default + name: MTEB StackOverflowQA (default) + revision: db8f169f3894c14a00251061f957b2063eef2bd5 + split: test + type: CoIR-Retrieval/stackoverflow-qa + metrics: + - type: ndcg_at_1 + value: 68.205 + - type: ndcg_at_3 + value: 75.473 + - type: ndcg_at_5 + value: 77.118 + - type: ndcg_at_10 + value: 78.45 + - type: ndcg_at_20 + value: 79.181 + - type: ndcg_at_100 + value: 80.259 + - type: ndcg_at_1000 + value: 80.518 + - type: map_at_1 + value: 68.205 + - type: map_at_3 + value: 73.763 + - type: map_at_5 + value: 74.68299999999999 + - type: map_at_10 + value: 75.234 + - type: map_at_20 + value: 75.43900000000001 + - type: map_at_100 + value: 75.59 + - type: map_at_1000 + value: 75.599 + - type: recall_at_1 + value: 68.205 + - type: recall_at_3 + value: 80.391 + - type: recall_at_5 + value: 84.353 + - type: recall_at_10 + value: 88.465 + - type: recall_at_20 + value: 91.32400000000001 + - type: recall_at_100 + value: 97.09100000000001 + - type: recall_at_1000 + value: 99.14699999999999 + - type: precision_at_1 + value: 68.205 + - type: precision_at_3 + value: 26.796999999999997 + - type: precision_at_5 + value: 16.871 + - type: precision_at_10 + value: 8.847 + - type: precision_at_20 + value: 4.566 + - type: precision_at_100 + value: 0.971 + - type: precision_at_1000 + value: 0.099 + - type: mrr_at_1 + value: 68.2046 + - type: mrr_at_3 + value: 73.763 + - type: mrr_at_5 + value: 74.6832 + - type: mrr_at_10 + value: 75.23440000000001 + - type: mrr_at_20 + value: 75.4389 + - type: mrr_at_100 + value: 75.5901 + - type: mrr_at_1000 + value: 75.59909999999999 + - type: nauc_ndcg_at_1_max + value: 70.0997 + - type: nauc_ndcg_at_1_std + value: -6.6174 + - type: nauc_ndcg_at_1_diff1 + value: 80.8018 + - type: nauc_ndcg_at_3_max + value: 71.8713 + - type: nauc_ndcg_at_3_std + value: -5.7584 + - type: nauc_ndcg_at_3_diff1 + value: 76.6152 + - type: nauc_ndcg_at_5_max + value: 71.7906 + - type: nauc_ndcg_at_5_std + value: -5.6573 + - type: nauc_ndcg_at_5_diff1 + value: 76.6923 + - type: nauc_ndcg_at_10_max + value: 71.4058 + - type: nauc_ndcg_at_10_std + value: -4.8043000000000005 + - type: nauc_ndcg_at_10_diff1 + value: 76.4267 + - type: nauc_ndcg_at_20_max + value: 71.5511 + - type: nauc_ndcg_at_20_std + value: -4.8308 + - type: nauc_ndcg_at_20_diff1 + value: 76.49669999999999 + - type: nauc_ndcg_at_100_max + value: 71.5604 + - type: nauc_ndcg_at_100_std + value: -4.8645000000000005 + - type: nauc_ndcg_at_100_diff1 + value: 77.022 + - type: nauc_ndcg_at_1000_max + value: 71.4953 + - type: nauc_ndcg_at_1000_std + value: -4.8631 + - type: nauc_ndcg_at_1000_diff1 + value: 77.1952 + - type: nauc_map_at_1_max + value: 70.0997 + - type: nauc_map_at_1_std + value: -6.6174 + - type: nauc_map_at_1_diff1 + value: 80.8018 + - type: nauc_map_at_3_max + value: 71.46329999999999 + - type: nauc_map_at_3_std + value: -5.9901 + - type: nauc_map_at_3_diff1 + value: 77.7281 + - type: nauc_map_at_5_max + value: 71.4046 + - type: nauc_map_at_5_std + value: -5.9794 + - type: nauc_map_at_5_diff1 + value: 77.8163 + - type: nauc_map_at_10_max + value: 71.2618 + - type: nauc_map_at_10_std + value: -5.702999999999999 + - type: nauc_map_at_10_diff1 + value: 77.73780000000001 + - type: nauc_map_at_20_max + value: 71.30330000000001 + - type: nauc_map_at_20_std + value: -5.691 + - type: nauc_map_at_20_diff1 + value: 77.7683 + - type: nauc_map_at_100_max + value: 71.3035 + - type: nauc_map_at_100_std + value: -5.680000000000001 + - type: nauc_map_at_100_diff1 + value: 77.8324 + - type: nauc_map_at_1000_max + value: 71.3013 + - type: nauc_map_at_1000_std + value: -5.6772 + - type: nauc_map_at_1000_diff1 + value: 77.837 + - type: nauc_recall_at_1_max + value: 70.0997 + - type: nauc_recall_at_1_std + value: -6.6174 + - type: nauc_recall_at_1_diff1 + value: 80.8018 + - type: nauc_recall_at_3_max + value: 73.3015 + - type: nauc_recall_at_3_std + value: -4.9247 + - type: nauc_recall_at_3_diff1 + value: 72.6201 + - type: nauc_recall_at_5_max + value: 73.3818 + - type: nauc_recall_at_5_std + value: -4.196 + - type: nauc_recall_at_5_diff1 + value: 71.8984 + - type: nauc_recall_at_10_max + value: 71.8002 + - type: nauc_recall_at_10_std + value: 1.0328 + - type: nauc_recall_at_10_diff1 + value: 69.0552 + - type: nauc_recall_at_20_max + value: 72.9934 + - type: nauc_recall_at_20_std + value: 2.0923000000000003 + - type: nauc_recall_at_20_diff1 + value: 67.3481 + - type: nauc_recall_at_100_max + value: 76.0971 + - type: nauc_recall_at_100_std + value: 12.4217 + - type: nauc_recall_at_100_diff1 + value: 66.6112 + - type: nauc_recall_at_1000_max + value: 76.7462 + - type: nauc_recall_at_1000_std + value: 50.754200000000004 + - type: nauc_recall_at_1000_diff1 + value: 69.8675 + - type: nauc_precision_at_1_max + value: 70.0997 + - type: nauc_precision_at_1_std + value: -6.6174 + - type: nauc_precision_at_1_diff1 + value: 80.8018 + - type: nauc_precision_at_3_max + value: 73.3015 + - type: nauc_precision_at_3_std + value: -4.9247 + - type: nauc_precision_at_3_diff1 + value: 72.6201 + - type: nauc_precision_at_5_max + value: 73.3818 + - type: nauc_precision_at_5_std + value: -4.196 + - type: nauc_precision_at_5_diff1 + value: 71.8984 + - type: nauc_precision_at_10_max + value: 71.8002 + - type: nauc_precision_at_10_std + value: 1.0328 + - type: nauc_precision_at_10_diff1 + value: 69.0552 + - type: nauc_precision_at_20_max + value: 72.9934 + - type: nauc_precision_at_20_std + value: 2.0923000000000003 + - type: nauc_precision_at_20_diff1 + value: 67.3481 + - type: nauc_precision_at_100_max + value: 76.0971 + - type: nauc_precision_at_100_std + value: 12.4217 + - type: nauc_precision_at_100_diff1 + value: 66.6112 + - type: nauc_precision_at_1000_max + value: 76.7462 + - type: nauc_precision_at_1000_std + value: 50.754200000000004 + - type: nauc_precision_at_1000_diff1 + value: 69.8675 + - type: nauc_mrr_at_1_max + value: 70.0997 + - type: nauc_mrr_at_1_std + value: -6.6174 + - type: nauc_mrr_at_1_diff1 + value: 80.8018 + - type: nauc_mrr_at_3_max + value: 71.46329999999999 + - type: nauc_mrr_at_3_std + value: -5.9901 + - type: nauc_mrr_at_3_diff1 + value: 77.7281 + - type: nauc_mrr_at_5_max + value: 71.4046 + - type: nauc_mrr_at_5_std + value: -5.9794 + - type: nauc_mrr_at_5_diff1 + value: 77.8163 + - type: nauc_mrr_at_10_max + value: 71.2618 + - type: nauc_mrr_at_10_std + value: -5.702999999999999 + - type: nauc_mrr_at_10_diff1 + value: 77.73780000000001 + - type: nauc_mrr_at_20_max + value: 71.30330000000001 + - type: nauc_mrr_at_20_std + value: -5.691 + - type: nauc_mrr_at_20_diff1 + value: 77.7683 + - type: nauc_mrr_at_100_max + value: 71.3035 + - type: nauc_mrr_at_100_std + value: -5.680000000000001 + - type: nauc_mrr_at_100_diff1 + value: 77.8324 + - type: nauc_mrr_at_1000_max + value: 71.3013 + - type: nauc_mrr_at_1000_std + value: -5.6772 + - type: nauc_mrr_at_1000_diff1 + value: 77.837 + - type: main_score + value: 78.45 + task: + type: Retrieval + - dataset: + config: default + name: MTEB SummEval (default) + revision: cda12ad7615edc362dbf25a00fdd61d3b1eaf93c + split: test + type: mteb/summeval + metrics: + - type: pearson + value: 31.7097 + - type: spearman + value: 32.0256 + - type: cosine_spearman + value: 32.0256 + - type: cosine_pearson + value: 31.7097 + - type: dot_spearman + value: 32.0256 + - type: dot_pearson + value: 31.7097 + - type: main_score + value: 32.0256 + task: + type: Summarization + - dataset: + config: default + name: MTEB SyntheticText2SQL (default) + revision: 686b87296c3a0191b5d9415a00526c62db9fce09 + split: test + type: CoIR-Retrieval/synthetic-text2sql + metrics: + - type: ndcg_at_1 + value: 3.5549999999999997 + - type: ndcg_at_3 + value: 41.534 + - type: ndcg_at_5 + value: 44.847 + - type: ndcg_at_10 + value: 47.344 + - type: ndcg_at_20 + value: 48.826 + - type: ndcg_at_100 + value: 50.442 + - type: ndcg_at_1000 + value: 50.937 + - type: map_at_1 + value: 3.5549999999999997 + - type: map_at_3 + value: 33.083 + - type: map_at_5 + value: 34.928 + - type: map_at_10 + value: 35.964 + - type: map_at_20 + value: 36.376 + - type: map_at_100 + value: 36.61 + - type: map_at_1000 + value: 36.63 + - type: recall_at_1 + value: 3.5549999999999997 + - type: recall_at_3 + value: 65.63 + - type: recall_at_5 + value: 73.646 + - type: recall_at_10 + value: 81.337 + - type: recall_at_20 + value: 87.165 + - type: recall_at_100 + value: 95.71 + - type: recall_at_1000 + value: 99.556 + - type: precision_at_1 + value: 3.5549999999999997 + - type: precision_at_3 + value: 21.877 + - type: precision_at_5 + value: 14.729000000000001 + - type: precision_at_10 + value: 8.134 + - type: precision_at_20 + value: 4.358 + - type: precision_at_100 + value: 0.9570000000000001 + - type: precision_at_1000 + value: 0.1 + - type: mrr_at_1 + value: 31.721100000000003 + - type: mrr_at_3 + value: 48.6754 + - type: mrr_at_5 + value: 50.3093 + - type: mrr_at_10 + value: 51.2454 + - type: mrr_at_20 + value: 51.629999999999995 + - type: mrr_at_100 + value: 51.8552 + - type: mrr_at_1000 + value: 51.8747 + - type: nauc_ndcg_at_1_max + value: 6.543 + - type: nauc_ndcg_at_1_std + value: -11.0614 + - type: nauc_ndcg_at_1_diff1 + value: 77.4191 + - type: nauc_ndcg_at_3_max + value: 35.9842 + - type: nauc_ndcg_at_3_std + value: -16.258200000000002 + - type: nauc_ndcg_at_3_diff1 + value: -62.2219 + - type: nauc_ndcg_at_5_max + value: 35.0885 + - type: nauc_ndcg_at_5_std + value: -14.935699999999999 + - type: nauc_ndcg_at_5_diff1 + value: -58.3931 + - type: nauc_ndcg_at_10_max + value: 33.7926 + - type: nauc_ndcg_at_10_std + value: -14.2862 + - type: nauc_ndcg_at_10_diff1 + value: -55.5325 + - type: nauc_ndcg_at_20_max + value: 33.631899999999995 + - type: nauc_ndcg_at_20_std + value: -14.061499999999999 + - type: nauc_ndcg_at_20_diff1 + value: -53.7148 + - type: nauc_ndcg_at_100_max + value: 32.736900000000006 + - type: nauc_ndcg_at_100_std + value: -13.7486 + - type: nauc_ndcg_at_100_diff1 + value: -52.0744 + - type: nauc_ndcg_at_1000_max + value: 32.941500000000005 + - type: nauc_ndcg_at_1000_std + value: -14.186099999999998 + - type: nauc_ndcg_at_1000_diff1 + value: -51.6402 + - type: nauc_map_at_1_max + value: 6.543 + - type: nauc_map_at_1_std + value: -11.0614 + - type: nauc_map_at_1_diff1 + value: 77.4191 + - type: nauc_map_at_3_max + value: 33.901399999999995 + - type: nauc_map_at_3_std + value: -15.789 + - type: nauc_map_at_3_diff1 + value: -53.5257 + - type: nauc_map_at_5_max + value: 33.1725 + - type: nauc_map_at_5_std + value: -14.948400000000001 + - type: nauc_map_at_5_diff1 + value: -50.5361 + - type: nauc_map_at_10_max + value: 32.5273 + - type: nauc_map_at_10_std + value: -14.648 + - type: nauc_map_at_10_diff1 + value: -48.928 + - type: nauc_map_at_20_max + value: 32.4474 + - type: nauc_map_at_20_std + value: -14.6155 + - type: nauc_map_at_20_diff1 + value: -48.2673 + - type: nauc_map_at_100_max + value: 32.2692 + - type: nauc_map_at_100_std + value: -14.5789 + - type: nauc_map_at_100_diff1 + value: -47.9677 + - type: nauc_map_at_1000_max + value: 32.2805 + - type: nauc_map_at_1000_std + value: -14.594999999999999 + - type: nauc_map_at_1000_diff1 + value: -47.944700000000005 + - type: nauc_recall_at_1_max + value: 6.543 + - type: nauc_recall_at_1_std + value: -11.0614 + - type: nauc_recall_at_1_diff1 + value: 77.4191 + - type: nauc_recall_at_3_max + value: 39.704899999999995 + - type: nauc_recall_at_3_std + value: -17.1274 + - type: nauc_recall_at_3_diff1 + value: -77.3937 + - type: nauc_recall_at_5_max + value: 38.8786 + - type: nauc_recall_at_5_std + value: -14.7304 + - type: nauc_recall_at_5_diff1 + value: -73.366 + - type: nauc_recall_at_10_max + value: 36.2642 + - type: nauc_recall_at_10_std + value: -12.828800000000001 + - type: nauc_recall_at_10_diff1 + value: -69.7955 + - type: nauc_recall_at_20_max + value: 36.5493 + - type: nauc_recall_at_20_std + value: -10.9359 + - type: nauc_recall_at_20_diff1 + value: -66.8099 + - type: nauc_recall_at_100_max + value: 29.1291 + - type: nauc_recall_at_100_std + value: 0.3365 + - type: nauc_recall_at_100_diff1 + value: -63.8938 + - type: nauc_recall_at_1000_max + value: 37.589800000000004 + - type: nauc_recall_at_1000_std + value: 17.3579 + - type: nauc_recall_at_1000_diff1 + value: -68.429 + - type: nauc_precision_at_1_max + value: 6.543 + - type: nauc_precision_at_1_std + value: -11.0614 + - type: nauc_precision_at_1_diff1 + value: 77.4191 + - type: nauc_precision_at_3_max + value: 39.704899999999995 + - type: nauc_precision_at_3_std + value: -17.1274 + - type: nauc_precision_at_3_diff1 + value: -77.3937 + - type: nauc_precision_at_5_max + value: 38.8786 + - type: nauc_precision_at_5_std + value: -14.7304 + - type: nauc_precision_at_5_diff1 + value: -73.366 + - type: nauc_precision_at_10_max + value: 36.2642 + - type: nauc_precision_at_10_std + value: -12.828800000000001 + - type: nauc_precision_at_10_diff1 + value: -69.7955 + - type: nauc_precision_at_20_max + value: 36.5493 + - type: nauc_precision_at_20_std + value: -10.9359 + - type: nauc_precision_at_20_diff1 + value: -66.8099 + - type: nauc_precision_at_100_max + value: 29.1291 + - type: nauc_precision_at_100_std + value: 0.3365 + - type: nauc_precision_at_100_diff1 + value: -63.8938 + - type: nauc_precision_at_1000_max + value: 37.589800000000004 + - type: nauc_precision_at_1000_std + value: 17.3579 + - type: nauc_precision_at_1000_diff1 + value: -68.429 + - type: nauc_mrr_at_1_max + value: 18.7616 + - type: nauc_mrr_at_1_std + value: -9.332600000000001 + - type: nauc_mrr_at_1_diff1 + value: -38.775 + - type: nauc_mrr_at_3_max + value: 27.9627 + - type: nauc_mrr_at_3_std + value: -12.1163 + - type: nauc_mrr_at_3_diff1 + value: -56.172900000000006 + - type: nauc_mrr_at_5_max + value: 27.385900000000003 + - type: nauc_mrr_at_5_std + value: -11.7823 + - type: nauc_mrr_at_5_diff1 + value: -55.085300000000004 + - type: nauc_mrr_at_10_max + value: 26.9297 + - type: nauc_mrr_at_10_std + value: -11.5899 + - type: nauc_mrr_at_10_diff1 + value: -54.352900000000005 + - type: nauc_mrr_at_20_max + value: 26.8231 + - type: nauc_mrr_at_20_std + value: -11.5438 + - type: nauc_mrr_at_20_diff1 + value: -54.101 + - type: nauc_mrr_at_100_max + value: 26.6888 + - type: nauc_mrr_at_100_std + value: -11.5184 + - type: nauc_mrr_at_100_diff1 + value: -53.9839 + - type: nauc_mrr_at_1000_max + value: 26.691399999999998 + - type: nauc_mrr_at_1000_std + value: -11.5244 + - type: nauc_mrr_at_1000_diff1 + value: -53.976 + - type: main_score + value: 47.344 + task: + type: Retrieval + - dataset: + config: default + name: MTEB TRECCOVID (default) + revision: bb9466bac8153a0349341eb1b22e06409e78ef4e + split: test + type: mteb/trec-covid + metrics: + - type: ndcg_at_1 + value: 70.0 + - type: ndcg_at_3 + value: 70.877 + - type: ndcg_at_5 + value: 70.735 + - type: ndcg_at_10 + value: 68.573 + - type: ndcg_at_20 + value: 65.635 + - type: ndcg_at_100 + value: 53.501 + - type: ndcg_at_1000 + value: 49.288 + - type: map_at_1 + value: 0.207 + - type: map_at_3 + value: 0.551 + - type: map_at_5 + value: 0.8909999999999999 + - type: map_at_10 + value: 1.635 + - type: map_at_20 + value: 2.952 + - type: map_at_100 + value: 9.713 + - type: map_at_1000 + value: 24.064 + - type: recall_at_1 + value: 0.207 + - type: recall_at_3 + value: 0.602 + - type: recall_at_5 + value: 0.992 + - type: recall_at_10 + value: 1.9009999999999998 + - type: recall_at_20 + value: 3.5709999999999997 + - type: recall_at_100 + value: 13.297999999999998 + - type: recall_at_1000 + value: 47.067 + - type: precision_at_1 + value: 80.0 + - type: precision_at_3 + value: 76.667 + - type: precision_at_5 + value: 76.4 + - type: precision_at_10 + value: 73.2 + - type: precision_at_20 + value: 70.1 + - type: precision_at_100 + value: 55.04 + - type: precision_at_1000 + value: 22.046 + - type: mrr_at_1 + value: 80.0 + - type: mrr_at_3 + value: 88.66669999999999 + - type: mrr_at_5 + value: 89.16669999999999 + - type: mrr_at_10 + value: 89.16669999999999 + - type: mrr_at_20 + value: 89.16669999999999 + - type: mrr_at_100 + value: 89.16669999999999 + - type: mrr_at_1000 + value: 89.16669999999999 + - type: nauc_ndcg_at_1_max + value: 9.0505 + - type: nauc_ndcg_at_1_std + value: 17.7341 + - type: nauc_ndcg_at_1_diff1 + value: -17.272399999999998 + - type: nauc_ndcg_at_3_max + value: 27.3702 + - type: nauc_ndcg_at_3_std + value: 43.432500000000005 + - type: nauc_ndcg_at_3_diff1 + value: -5.716600000000001 + - type: nauc_ndcg_at_5_max + value: 24.6447 + - type: nauc_ndcg_at_5_std + value: 48.0114 + - type: nauc_ndcg_at_5_diff1 + value: -7.0447999999999995 + - type: nauc_ndcg_at_10_max + value: 31.5589 + - type: nauc_ndcg_at_10_std + value: 60.242 + - type: nauc_ndcg_at_10_diff1 + value: -4.827 + - type: nauc_ndcg_at_20_max + value: 39.195600000000006 + - type: nauc_ndcg_at_20_std + value: 67.9313 + - type: nauc_ndcg_at_20_diff1 + value: -10.0317 + - type: nauc_ndcg_at_100_max + value: 43.8896 + - type: nauc_ndcg_at_100_std + value: 76.6623 + - type: nauc_ndcg_at_100_diff1 + value: -14.7694 + - type: nauc_ndcg_at_1000_max + value: 46.935 + - type: nauc_ndcg_at_1000_std + value: 79.9247 + - type: nauc_ndcg_at_1000_diff1 + value: -12.9885 + - type: nauc_map_at_1_max + value: 5.587899999999999 + - type: nauc_map_at_1_std + value: -6.5333000000000006 + - type: nauc_map_at_1_diff1 + value: 7.8414 + - type: nauc_map_at_3_max + value: 14.21 + - type: nauc_map_at_3_std + value: 7.9614 + - type: nauc_map_at_3_diff1 + value: 11.9467 + - type: nauc_map_at_5_max + value: 14.514299999999999 + - type: nauc_map_at_5_std + value: 10.6974 + - type: nauc_map_at_5_diff1 + value: 11.732800000000001 + - type: nauc_map_at_10_max + value: 17.5629 + - type: nauc_map_at_10_std + value: 21.4707 + - type: nauc_map_at_10_diff1 + value: 10.9138 + - type: nauc_map_at_20_max + value: 23.891399999999997 + - type: nauc_map_at_20_std + value: 32.5254 + - type: nauc_map_at_20_diff1 + value: 5.6072999999999995 + - type: nauc_map_at_100_max + value: 37.247 + - type: nauc_map_at_100_std + value: 66.2197 + - type: nauc_map_at_100_diff1 + value: -6.0896 + - type: nauc_map_at_1000_max + value: 51.590599999999995 + - type: nauc_map_at_1000_std + value: 83.3358 + - type: nauc_map_at_1000_diff1 + value: -18.7689 + - type: nauc_recall_at_1_max + value: 5.587899999999999 + - type: nauc_recall_at_1_std + value: -6.5333000000000006 + - type: nauc_recall_at_1_diff1 + value: 7.8414 + - type: nauc_recall_at_3_max + value: 10.6036 + - type: nauc_recall_at_3_std + value: 8.7269 + - type: nauc_recall_at_3_diff1 + value: 13.296 + - type: nauc_recall_at_5_max + value: 9.3121 + - type: nauc_recall_at_5_std + value: 9.9978 + - type: nauc_recall_at_5_diff1 + value: 12.5994 + - type: nauc_recall_at_10_max + value: 10.0265 + - type: nauc_recall_at_10_std + value: 16.8073 + - type: nauc_recall_at_10_diff1 + value: 10.8776 + - type: nauc_recall_at_20_max + value: 16.3788 + - type: nauc_recall_at_20_std + value: 23.7003 + - type: nauc_recall_at_20_diff1 + value: 7.832 + - type: nauc_recall_at_100_max + value: 25.289 + - type: nauc_recall_at_100_std + value: 51.6757 + - type: nauc_recall_at_100_diff1 + value: 0.4044 + - type: nauc_recall_at_1000_max + value: 42.1531 + - type: nauc_recall_at_1000_std + value: 72.10419999999999 + - type: nauc_recall_at_1000_diff1 + value: -12.410499999999999 + - type: nauc_precision_at_1_max + value: 31.203799999999998 + - type: nauc_precision_at_1_std + value: 23.1918 + - type: nauc_precision_at_1_diff1 + value: -32.057900000000004 + - type: nauc_precision_at_3_max + value: 40.368300000000005 + - type: nauc_precision_at_3_std + value: 50.225699999999996 + - type: nauc_precision_at_3_diff1 + value: -2.2047 + - type: nauc_precision_at_5_max + value: 29.592200000000002 + - type: nauc_precision_at_5_std + value: 49.6822 + - type: nauc_precision_at_5_diff1 + value: -4.1202000000000005 + - type: nauc_precision_at_10_max + value: 41.876400000000004 + - type: nauc_precision_at_10_std + value: 67.3955 + - type: nauc_precision_at_10_diff1 + value: 1.8023 + - type: nauc_precision_at_20_max + value: 49.011500000000005 + - type: nauc_precision_at_20_std + value: 72.0322 + - type: nauc_precision_at_20_diff1 + value: -8.0818 + - type: nauc_precision_at_100_max + value: 49.385200000000005 + - type: nauc_precision_at_100_std + value: 79.20660000000001 + - type: nauc_precision_at_100_diff1 + value: -12.9969 + - type: nauc_precision_at_1000_max + value: 41.5596 + - type: nauc_precision_at_1000_std + value: 51.89470000000001 + - type: nauc_precision_at_1000_diff1 + value: -24.5507 + - type: nauc_mrr_at_1_max + value: 31.203799999999998 + - type: nauc_mrr_at_1_std + value: 23.1918 + - type: nauc_mrr_at_1_diff1 + value: -32.057900000000004 + - type: nauc_mrr_at_3_max + value: 37.7018 + - type: nauc_mrr_at_3_std + value: 31.9141 + - type: nauc_mrr_at_3_diff1 + value: -22.4835 + - type: nauc_mrr_at_5_max + value: 35.284 + - type: nauc_mrr_at_5_std + value: 28.569899999999997 + - type: nauc_mrr_at_5_diff1 + value: -26.309700000000003 + - type: nauc_mrr_at_10_max + value: 35.284 + - type: nauc_mrr_at_10_std + value: 28.569899999999997 + - type: nauc_mrr_at_10_diff1 + value: -26.309700000000003 + - type: nauc_mrr_at_20_max + value: 35.284 + - type: nauc_mrr_at_20_std + value: 28.569899999999997 + - type: nauc_mrr_at_20_diff1 + value: -26.309700000000003 + - type: nauc_mrr_at_100_max + value: 35.284 + - type: nauc_mrr_at_100_std + value: 28.569899999999997 + - type: nauc_mrr_at_100_diff1 + value: -26.309700000000003 + - type: nauc_mrr_at_1000_max + value: 35.284 + - type: nauc_mrr_at_1000_std + value: 28.569899999999997 + - type: nauc_mrr_at_1000_diff1 + value: -26.309700000000003 + - type: main_score + value: 68.573 + task: + type: Retrieval + - dataset: + config: default + name: MTEB Touche2020 (default) + revision: a34f9a33db75fa0cbb21bb5cfc3dae8dc8bec93f + split: test + type: mteb/touche2020 + metrics: + - type: ndcg_at_1 + value: 41.837 + - type: ndcg_at_3 + value: 34.675 + - type: ndcg_at_5 + value: 30.017 + - type: ndcg_at_10 + value: 27.306 + - type: ndcg_at_20 + value: 27.009 + - type: ndcg_at_100 + value: 38.037 + - type: ndcg_at_1000 + value: 49.413000000000004 + - type: map_at_1 + value: 3.304 + - type: map_at_3 + value: 6.0569999999999995 + - type: map_at_5 + value: 7.856000000000001 + - type: map_at_10 + value: 10.869 + - type: map_at_20 + value: 12.824 + - type: map_at_100 + value: 16.631999999999998 + - type: map_at_1000 + value: 18.138 + - type: recall_at_1 + value: 3.304 + - type: recall_at_3 + value: 7.13 + - type: recall_at_5 + value: 9.995999999999999 + - type: recall_at_10 + value: 16.766000000000002 + - type: recall_at_20 + value: 22.933 + - type: recall_at_100 + value: 47.427 + - type: recall_at_1000 + value: 81.527 + - type: precision_at_1 + value: 42.857 + - type: precision_at_3 + value: 35.374 + - type: precision_at_5 + value: 28.163 + - type: precision_at_10 + value: 23.061 + - type: precision_at_20 + value: 16.633 + - type: precision_at_100 + value: 7.632999999999999 + - type: precision_at_1000 + value: 1.51 + - type: mrr_at_1 + value: 42.857099999999996 + - type: mrr_at_3 + value: 54.4218 + - type: mrr_at_5 + value: 54.4218 + - type: mrr_at_10 + value: 56.431 + - type: mrr_at_20 + value: 56.880900000000004 + - type: mrr_at_100 + value: 57.0526 + - type: mrr_at_1000 + value: 57.0526 + - type: nauc_ndcg_at_1_max + value: -44.2104 + - type: nauc_ndcg_at_1_std + value: -2.3875 + - type: nauc_ndcg_at_1_diff1 + value: -23.4197 + - type: nauc_ndcg_at_3_max + value: -40.1986 + - type: nauc_ndcg_at_3_std + value: -4.3845 + - type: nauc_ndcg_at_3_diff1 + value: -26.881100000000004 + - type: nauc_ndcg_at_5_max + value: -37.8693 + - type: nauc_ndcg_at_5_std + value: -5.817 + - type: nauc_ndcg_at_5_diff1 + value: -30.292599999999997 + - type: nauc_ndcg_at_10_max + value: -35.0514 + - type: nauc_ndcg_at_10_std + value: -12.628 + - type: nauc_ndcg_at_10_diff1 + value: -28.5171 + - type: nauc_ndcg_at_20_max + value: -36.829499999999996 + - type: nauc_ndcg_at_20_std + value: -10.9047 + - type: nauc_ndcg_at_20_diff1 + value: -25.590200000000003 + - type: nauc_ndcg_at_100_max + value: -33.1224 + - type: nauc_ndcg_at_100_std + value: 14.3094 + - type: nauc_ndcg_at_100_diff1 + value: -17.6544 + - type: nauc_ndcg_at_1000_max + value: -30.8819 + - type: nauc_ndcg_at_1000_std + value: 22.3523 + - type: nauc_ndcg_at_1000_diff1 + value: -19.5741 + - type: nauc_map_at_1_max + value: -38.6863 + - type: nauc_map_at_1_std + value: -15.0366 + - type: nauc_map_at_1_diff1 + value: -8.5063 + - type: nauc_map_at_3_max + value: -38.9161 + - type: nauc_map_at_3_std + value: -16.71 + - type: nauc_map_at_3_diff1 + value: -21.3221 + - type: nauc_map_at_5_max + value: -35.0036 + - type: nauc_map_at_5_std + value: -18.4668 + - type: nauc_map_at_5_diff1 + value: -27.6758 + - type: nauc_map_at_10_max + value: -29.7816 + - type: nauc_map_at_10_std + value: -20.890900000000002 + - type: nauc_map_at_10_diff1 + value: -27.380100000000002 + - type: nauc_map_at_20_max + value: -29.3362 + - type: nauc_map_at_20_std + value: -18.9281 + - type: nauc_map_at_20_diff1 + value: -27.058500000000002 + - type: nauc_map_at_100_max + value: -27.9555 + - type: nauc_map_at_100_std + value: -7.222 + - type: nauc_map_at_100_diff1 + value: -22.7849 + - type: nauc_map_at_1000_max + value: -26.954 + - type: nauc_map_at_1000_std + value: -4.0097000000000005 + - type: nauc_map_at_1000_diff1 + value: -22.855 + - type: nauc_recall_at_1_max + value: -38.6863 + - type: nauc_recall_at_1_std + value: -15.0366 + - type: nauc_recall_at_1_diff1 + value: -8.5063 + - type: nauc_recall_at_3_max + value: -42.2532 + - type: nauc_recall_at_3_std + value: -20.399 + - type: nauc_recall_at_3_diff1 + value: -23.8415 + - type: nauc_recall_at_5_max + value: -35.3457 + - type: nauc_recall_at_5_std + value: -20.0969 + - type: nauc_recall_at_5_diff1 + value: -29.5907 + - type: nauc_recall_at_10_max + value: -31.7181 + - type: nauc_recall_at_10_std + value: -22.9559 + - type: nauc_recall_at_10_diff1 + value: -22.564400000000003 + - type: nauc_recall_at_20_max + value: -34.5273 + - type: nauc_recall_at_20_std + value: -15.6335 + - type: nauc_recall_at_20_diff1 + value: -22.9889 + - type: nauc_recall_at_100_max + value: -28.2509 + - type: nauc_recall_at_100_std + value: 30.481399999999997 + - type: nauc_recall_at_100_diff1 + value: -6.9437999999999995 + - type: nauc_recall_at_1000_max + value: -12.5952 + - type: nauc_recall_at_1000_std + value: 69.9957 + - type: nauc_recall_at_1000_diff1 + value: 2.2129 + - type: nauc_precision_at_1_max + value: -45.3657 + - type: nauc_precision_at_1_std + value: -4.4435 + - type: nauc_precision_at_1_diff1 + value: -18.6647 + - type: nauc_precision_at_3_max + value: -39.1078 + - type: nauc_precision_at_3_std + value: -8.047600000000001 + - type: nauc_precision_at_3_diff1 + value: -27.322200000000002 + - type: nauc_precision_at_5_max + value: -32.8848 + - type: nauc_precision_at_5_std + value: -8.5508 + - type: nauc_precision_at_5_diff1 + value: -31.567600000000002 + - type: nauc_precision_at_10_max + value: -28.719499999999996 + - type: nauc_precision_at_10_std + value: -14.498800000000001 + - type: nauc_precision_at_10_diff1 + value: -27.8402 + - type: nauc_precision_at_20_max + value: -26.466 + - type: nauc_precision_at_20_std + value: 3.3133000000000004 + - type: nauc_precision_at_20_diff1 + value: -31.5367 + - type: nauc_precision_at_100_max + value: -5.4186 + - type: nauc_precision_at_100_std + value: 61.58709999999999 + - type: nauc_precision_at_100_diff1 + value: -8.8049 + - type: nauc_precision_at_1000_max + value: 37.745400000000004 + - type: nauc_precision_at_1000_std + value: 48.7776 + - type: nauc_precision_at_1000_diff1 + value: 6.4595 + - type: nauc_mrr_at_1_max + value: -45.3657 + - type: nauc_mrr_at_1_std + value: -4.4435 + - type: nauc_mrr_at_1_diff1 + value: -18.6647 + - type: nauc_mrr_at_3_max + value: -52.9035 + - type: nauc_mrr_at_3_std + value: -13.174800000000001 + - type: nauc_mrr_at_3_diff1 + value: -20.045299999999997 + - type: nauc_mrr_at_5_max + value: -52.9035 + - type: nauc_mrr_at_5_std + value: -13.174800000000001 + - type: nauc_mrr_at_5_diff1 + value: -20.045299999999997 + - type: nauc_mrr_at_10_max + value: -51.358599999999996 + - type: nauc_mrr_at_10_std + value: -11.266 + - type: nauc_mrr_at_10_diff1 + value: -19.4274 + - type: nauc_mrr_at_20_max + value: -51.648799999999994 + - type: nauc_mrr_at_20_std + value: -10.9663 + - type: nauc_mrr_at_20_diff1 + value: -19.5931 + - type: nauc_mrr_at_100_max + value: -51.669200000000004 + - type: nauc_mrr_at_100_std + value: -10.9424 + - type: nauc_mrr_at_100_diff1 + value: -19.7412 + - type: nauc_mrr_at_1000_max + value: -51.669200000000004 + - type: nauc_mrr_at_1000_std + value: -10.9424 + - type: nauc_mrr_at_1000_diff1 + value: -19.7412 + - type: main_score + value: 27.306 + task: + type: Retrieval + - dataset: + config: default + name: MTEB ToxicConversationsClassification (default) + revision: edfaf9da55d3dd50d43143d90c1ac476895ae6de + split: test + type: mteb/toxic_conversations_50k + metrics: + - type: accuracy + value: 62.480500000000006 + - type: f1 + value: 48.201100000000004 + - type: f1_weighted + value: 70.8591 + - type: ap + value: 10.9948 + - type: ap_weighted + value: 10.9948 + - type: main_score + value: 62.480500000000006 + task: + type: Classification + - dataset: + config: default + name: MTEB TweetSentimentExtractionClassification (default) + revision: d604517c81ca91fe16a244d1248fc021f9ecee7a + split: test + type: mteb/tweet_sentiment_extraction + metrics: + - type: accuracy + value: 58.3616 + - type: f1 + value: 58.5596 + - type: f1_weighted + value: 57.801 + - type: main_score + value: 58.3616 + task: + type: Classification + - dataset: + config: default + name: MTEB TwentyNewsgroupsClustering (default) + revision: 6125ec4e24fa026cec8a478383ee943acfbd5449 + split: test + type: mteb/twentynewsgroups-clustering + metrics: + - type: v_measure + value: 38.6199 + - type: v_measure_std + value: 2.3855999999999997 + - type: main_score + value: 38.6199 + task: + type: Clustering + - dataset: + config: default + name: MTEB TwitterSemEval2015 (default) + revision: 70970daeab8776df92f5ea462b6173c0b46fd2d1 + split: test + type: mteb/twittersemeval2015-pairclassification + metrics: + - type: similarity_accuracy + value: 82.9886 + - type: similarity_accuracy_threshold + value: 86.3901 + - type: similarity_f1 + value: 60.866200000000006 + - type: similarity_f1_threshold + value: 83.9821 + - type: similarity_precision + value: 59.333499999999994 + - type: similarity_recall + value: 62.480199999999996 + - type: similarity_ap + value: 64.413 + - type: cosine_accuracy + value: 82.9886 + - type: cosine_accuracy_threshold + value: 86.3901 + - type: cosine_f1 + value: 60.866200000000006 + - type: cosine_f1_threshold + value: 83.9821 + - type: cosine_precision + value: 59.333499999999994 + - type: cosine_recall + value: 62.480199999999996 + - type: cosine_ap + value: 64.413 + - type: manhattan_accuracy + value: 82.9409 + - type: manhattan_accuracy_threshold + value: 1144.7468000000001 + - type: manhattan_f1 + value: 60.760400000000004 + - type: manhattan_f1_threshold + value: 1291.7232999999999 + - type: manhattan_precision + value: 54.7126 + - type: manhattan_recall + value: 68.3113 + - type: manhattan_ap + value: 64.3592 + - type: euclidean_accuracy + value: 82.9886 + - type: euclidean_accuracy_threshold + value: 52.1726 + - type: euclidean_f1 + value: 60.866200000000006 + - type: euclidean_f1_threshold + value: 56.6001 + - type: euclidean_precision + value: 59.333499999999994 + - type: euclidean_recall + value: 62.480199999999996 + - type: euclidean_ap + value: 64.4131 + - type: dot_accuracy + value: 82.9886 + - type: dot_accuracy_threshold + value: 86.3901 + - type: dot_f1 + value: 60.866200000000006 + - type: dot_f1_threshold + value: 83.9821 + - type: dot_precision + value: 59.333499999999994 + - type: dot_recall + value: 62.480199999999996 + - type: dot_ap + value: 64.413 + - type: max_accuracy + value: 82.9886 + - type: max_f1 + value: 60.866200000000006 + - type: max_precision + value: 59.333499999999994 + - type: max_recall + value: 68.3113 + - type: max_ap + value: 64.4131 + - type: main_score + value: 64.4131 + task: + type: PairClassification + - dataset: + config: default + name: MTEB TwitterURLCorpus (default) + revision: 8b6510b0b1fa4e4c4f879467980e9be563ec1cdf + split: test + type: mteb/twitterurlcorpus-pairclassification + metrics: + - type: similarity_accuracy + value: 88.95100000000001 + - type: similarity_accuracy_threshold + value: 82.18520000000001 + - type: similarity_f1 + value: 77.9051 + - type: similarity_f1_threshold + value: 80.3369 + - type: similarity_precision + value: 76.07310000000001 + - type: similarity_recall + value: 79.8275 + - type: similarity_ap + value: 86.1545 + - type: cosine_accuracy + value: 88.95100000000001 + - type: cosine_accuracy_threshold + value: 82.18520000000001 + - type: cosine_f1 + value: 77.9051 + - type: cosine_f1_threshold + value: 80.3369 + - type: cosine_precision + value: 76.07310000000001 + - type: cosine_recall + value: 79.8275 + - type: cosine_ap + value: 86.1545 + - type: manhattan_accuracy + value: 88.9277 + - type: manhattan_accuracy_threshold + value: 1338.2836 + - type: manhattan_f1 + value: 77.8186 + - type: manhattan_f1_threshold + value: 1372.5978 + - type: manhattan_precision + value: 76.5745 + - type: manhattan_recall + value: 79.1038 + - type: manhattan_ap + value: 86.114 + - type: euclidean_accuracy + value: 88.95100000000001 + - type: euclidean_accuracy_threshold + value: 59.6905 + - type: euclidean_f1 + value: 77.9051 + - type: euclidean_f1_threshold + value: 62.71060000000001 + - type: euclidean_precision + value: 76.07310000000001 + - type: euclidean_recall + value: 79.8275 + - type: euclidean_ap + value: 86.1544 + - type: dot_accuracy + value: 88.95100000000001 + - type: dot_accuracy_threshold + value: 82.18520000000001 + - type: dot_f1 + value: 77.9051 + - type: dot_f1_threshold + value: 80.3369 + - type: dot_precision + value: 76.07310000000001 + - type: dot_recall + value: 79.8275 + - type: dot_ap + value: 86.1544 + - type: max_accuracy + value: 88.95100000000001 + - type: max_f1 + value: 77.9051 + - type: max_precision + value: 76.5745 + - type: max_recall + value: 79.8275 + - type: max_ap + value: 86.1545 + - type: main_score + value: 86.1545 + task: + type: PairClassification pipeline_tag: sentence-similarity --- # Granite-Embedding-278m-multilingual