Add new SentenceTransformer model

1_Pooling/config.json

@@ -1,5 +1,5 @@
 {
-  "word_embedding_dimension": 384,
+  "word_embedding_dimension": 768,
   "pooling_mode_cls_token": false,
   "pooling_mode_mean_tokens": true,
   "pooling_mode_max_tokens": false,

README.md

@@ -8,7 +8,7 @@ tags:
 - generated_from_trainer
 - dataset_size:53224
 - loss:MultipleNegativesRankingLoss
-base_model: sentence-transformers/all-MiniLM-L6-v2
+base_model: sentence-transformers/all-mpnet-base-v2
 widget:
 - source_sentence: ' A juridical person may not be a partner of a civil law union. '
   sentences:
@@ -220,7 +220,7 @@ metrics:
 - cosine_mrr@10
 - cosine_map@100
 model-index:
-- name: SentenceTransformer based on sentence-transformers/all-MiniLM-L6-v2
+- name: SentenceTransformer based on sentence-transformers/all-mpnet-base-v2
   results:
   - task:
       type: information-retrieval
@@ -230,49 +230,49 @@ model-index:
       type: mteb/AILA_casedocs
     metrics:
     - type: cosine_accuracy@1
-      value: 0.26
+      value: 0.24
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.36
+      value: 0.34
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.38
+      value: 0.4
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.58
+      value: 0.52
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.26
+      value: 0.24
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.2
+      value: 0.16666666666666663
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.14
+      value: 0.136
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.10599999999999998
+      value: 0.094
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.08253846153846153
+      value: 0.06678088578088578
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.183986013986014
+      value: 0.1388193473193473
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.21322843822843823
+      value: 0.18372843822843823
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.30445687645687647
+      value: 0.2667284382284382
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.261956835808035
+      value: 0.22218705752805715
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.3361349206349206
+      value: 0.3134126984126984
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.23084417119066455
+      value: 0.18539536890113958
       name: Cosine Map@100
   - task:
       type: information-retrieval
@@ -282,49 +282,49 @@ model-index:
       type: mteb/AILA_statutes
     metrics:
     - type: cosine_accuracy@1
-      value: 0.26
+      value: 0.28
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.44
+      value: 0.58
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.54
+      value: 0.8
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.7
+      value: 0.9
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.26
+      value: 0.28
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.16666666666666669
+      value: 0.22666666666666668
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.14400000000000002
+      value: 0.22399999999999998
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.10999999999999999
+      value: 0.15799999999999997
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.071
+      value: 0.073
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.129
+      value: 0.17266666666666666
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.17700000000000002
+      value: 0.2763333333333334
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.2643333333333333
+      value: 0.3773333333333333
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.23332317287231785
+      value: 0.32396168684748544
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.37441269841269836
+      value: 0.48524603174603165
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.2043241006581302
+      value: 0.26147750527977026
       name: Cosine Map@100
   - task:
       type: information-retrieval
@@ -334,49 +334,49 @@ model-index:
       type: mteb/legalbench_consumer_contracts_qa
     metrics:
     - type: cosine_accuracy@1
-      value: 0.45202020202020204
+      value: 0.4292929292929293
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.6868686868686869
+      value: 0.6363636363636364
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.7878787878787878
+      value: 0.7095959595959596
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.8737373737373737
+      value: 0.8156565656565656
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.45202020202020204
+      value: 0.4292929292929293
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.22895622895622894
+      value: 0.21212121212121207
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.15757575757575756
+      value: 0.1419191919191919
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.08737373737373735
+      value: 0.08156565656565656
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.45202020202020204
+      value: 0.4292929292929293
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.6868686868686869
+      value: 0.6363636363636364
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.7878787878787878
+      value: 0.7095959595959596
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.8737373737373737
+      value: 0.8156565656565656
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.660855212722782
+      value: 0.6114603730669577
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.5928561407728073
+      value: 0.5472532868366202
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.5987644318492056
+      value: 0.555387361338846
       name: Cosine Map@100
   - task:
       type: information-retrieval
@@ -386,49 +386,49 @@ model-index:
       type: mteb/legalbench_corporate_lobbying
     metrics:
     - type: cosine_accuracy@1
-      value: 0.7705882352941177
+      value: 0.6441176470588236
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.9088235294117647
+      value: 0.8558823529411764
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.9382352941176471
+      value: 0.8823529411764706
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.9705882352941176
+      value: 0.9147058823529411
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.7705882352941177
+      value: 0.6441176470588236
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.3029411764705882
+      value: 0.2852941176470588
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.18764705882352936
+      value: 0.17647058823529413
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.09705882352941174
+      value: 0.09147058823529411
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.7705882352941177
+      value: 0.6441176470588236
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.9088235294117647
+      value: 0.8558823529411764
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.9382352941176471
+      value: 0.8823529411764706
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.9705882352941176
+      value: 0.9147058823529411
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.877258980240739
+      value: 0.7924078571703878
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.8466806722689075
+      value: 0.751936274509804
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.8476651359451062
+      value: 0.754712212674935
       name: Cosine Map@100
   - task:
       type: information-retrieval
@@ -438,63 +438,63 @@ model-index:
       type: mteb/legal_summarization
     metrics:
     - type: cosine_accuracy@1
-      value: 0.4894366197183099
+      value: 0.41901408450704225
       name: Cosine Accuracy@1
     - type: cosine_accuracy@3
-      value: 0.6408450704225352
+      value: 0.5563380281690141
       name: Cosine Accuracy@3
     - type: cosine_accuracy@5
-      value: 0.7147887323943662
+      value: 0.6338028169014085
       name: Cosine Accuracy@5
     - type: cosine_accuracy@10
-      value: 0.7816901408450704
+      value: 0.7183098591549296
       name: Cosine Accuracy@10
     - type: cosine_precision@1
-      value: 0.4894366197183099
+      value: 0.41901408450704225
       name: Cosine Precision@1
     - type: cosine_precision@3
-      value: 0.23591549295774647
+      value: 0.20070422535211266
       name: Cosine Precision@3
     - type: cosine_precision@5
-      value: 0.16619718309859152
+      value: 0.14295774647887324
       name: Cosine Precision@5
     - type: cosine_precision@10
-      value: 0.09753521126760564
+      value: 0.08838028169014084
       name: Cosine Precision@10
     - type: cosine_recall@1
-      value: 0.4368868514114993
+      value: 0.35939538747637334
       name: Cosine Recall@1
     - type: cosine_recall@3
-      value: 0.5753959362234009
+      value: 0.4814835985610633
       name: Cosine Recall@3
     - type: cosine_recall@5
-      value: 0.6440091305408207
+      value: 0.5483042192549235
       name: Cosine Recall@5
     - type: cosine_recall@10
-      value: 0.7159090909090909
+      value: 0.6505441741357234
       name: Cosine Recall@10
     - type: cosine_ndcg@10
-      value: 0.596027060399293
+      value: 0.5155518221457815
       name: Cosine Ndcg@10
     - type: cosine_mrr@10
-      value: 0.5833137715179968
+      value: 0.5074348871003801
       name: Cosine Mrr@10
     - type: cosine_map@100
-      value: 0.5567992166327345
+      value: 0.46706462134757426
       name: Cosine Map@100
 ---
 
-# SentenceTransformer based on sentence-transformers/all-MiniLM-L6-v2
+# SentenceTransformer based on sentence-transformers/all-mpnet-base-v2
 
-This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) on the [coliee](https://huggingface.co/datasets/sentence-transformers/coliee), [legal_qa](https://huggingface.co/datasets/bwang0911/legal_qa_v1), [law_stack](https://huggingface.co/datasets/bwang0911/law_stackexchange), [legal_lens](https://huggingface.co/datasets/bwang0911/legal_lens_nli), [cuad_qa](https://huggingface.co/datasets/bwang0911/cuad_qa), [privacy_qa](https://huggingface.co/datasets/bwang0911/privacy_qa), [legal_sum](https://huggingface.co/datasets/bwang0911/legal_case_summarization) and [aus_legal_qa](https://huggingface.co/datasets/bwang0911/aus_legal_qa) datasets. It maps sentences & paragraphs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
+This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [sentence-transformers/all-mpnet-base-v2](https://huggingface.co/sentence-transformers/all-mpnet-base-v2) on the [coliee](https://huggingface.co/datasets/sentence-transformers/coliee), [legal_qa](https://huggingface.co/datasets/bwang0911/legal_qa_v1), [law_stack](https://huggingface.co/datasets/bwang0911/law_stackexchange), [legal_lens](https://huggingface.co/datasets/bwang0911/legal_lens_nli), [cuad_qa](https://huggingface.co/datasets/bwang0911/cuad_qa), [privacy_qa](https://huggingface.co/datasets/bwang0911/privacy_qa), [legal_sum](https://huggingface.co/datasets/bwang0911/legal_case_summarization) and [aus_legal_qa](https://huggingface.co/datasets/bwang0911/aus_legal_qa) datasets. It maps sentences & paragraphs to a 768-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.
 
 ## Model Details
 
 ### Model Description
 - **Model Type:** Sentence Transformer
-- **Base model:** [sentence-transformers/all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) <!-- at revision fa97f6e7cb1a59073dff9e6b13e2715cf7475ac9 -->
-- **Maximum Sequence Length:** 256 tokens
-- **Output Dimensionality:** 384 dimensions
+- **Base model:** [sentence-transformers/all-mpnet-base-v2](https://huggingface.co/sentence-transformers/all-mpnet-base-v2) <!-- at revision 9a3225965996d404b775526de6dbfe85d3368642 -->
+- **Maximum Sequence Length:** 192 tokens
+- **Output Dimensionality:** 768 dimensions
 - **Similarity Function:** Cosine Similarity
 - **Training Datasets:**
     - [coliee](https://huggingface.co/datasets/sentence-transformers/coliee)
@@ -518,8 +518,8 @@ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [s
 
 ```
 SentenceTransformer(
-  (0): Transformer({'max_seq_length': 256, 'do_lower_case': False}) with Transformer model: BertModel 
-  (1): Pooling({'word_embedding_dimension': 384, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
+  (0): Transformer({'max_seq_length': 192, 'do_lower_case': False}) with Transformer model: MPNetModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
   (2): Normalize()
 )
 ```
@@ -548,7 +548,7 @@ sentences = [
 ]
 embeddings = model.encode(sentences)
 print(embeddings.shape)
-# [3, 384]
+# [3, 768]
 
 # Get the similarity scores for the embeddings
 similarities = model.similarity(embeddings, embeddings)
@@ -591,21 +591,21 @@ You can finetune this model on your own dataset.
 
 | Metric              | mteb/AILA_casedocs | mteb/AILA_statutes | mteb/legalbench_consumer_contracts_qa | mteb/legalbench_corporate_lobbying | mteb/legal_summarization |
 |:--------------------|:-------------------|:-------------------|:--------------------------------------|:-----------------------------------|:-------------------------|
-| cosine_accuracy@1   | 0.26               | 0.26               | 0.452                                 | 0.7706                             | 0.4894                   |
-| cosine_accuracy@3   | 0.36               | 0.44               | 0.6869                                | 0.9088                             | 0.6408                   |
-| cosine_accuracy@5   | 0.38               | 0.54               | 0.7879                                | 0.9382                             | 0.7148                   |
-| cosine_accuracy@10  | 0.58               | 0.7                | 0.8737                                | 0.9706                             | 0.7817                   |
-| cosine_precision@1  | 0.26               | 0.26               | 0.452                                 | 0.7706                             | 0.4894                   |
-| cosine_precision@3  | 0.2                | 0.1667             | 0.229                                 | 0.3029                             | 0.2359                   |
-| cosine_precision@5  | 0.14               | 0.144              | 0.1576                                | 0.1876                             | 0.1662                   |
-| cosine_precision@10 | 0.106              | 0.11               | 0.0874                                | 0.0971                             | 0.0975                   |
-| cosine_recall@1     | 0.0825             | 0.071              | 0.452                                 | 0.7706                             | 0.4369                   |
-| cosine_recall@3     | 0.184              | 0.129              | 0.6869                                | 0.9088                             | 0.5754                   |
-| cosine_recall@5     | 0.2132             | 0.177              | 0.7879                                | 0.9382                             | 0.644                    |
-| cosine_recall@10    | 0.3045             | 0.2643             | 0.8737                                | 0.9706                             | 0.7159                   |
-| **cosine_ndcg@10**  | **0.262**          | **0.2333**         | **0.6609**                            | **0.8773**                         | **0.596**                |
-| cosine_mrr@10       | 0.3361             | 0.3744             | 0.5929                                | 0.8467                             | 0.5833                   |
-| cosine_map@100      | 0.2308             | 0.2043             | 0.5988                                | 0.8477                             | 0.5568                   |
+| cosine_accuracy@1   | 0.24               | 0.28               | 0.4293                                | 0.6441                             | 0.419                    |
+| cosine_accuracy@3   | 0.34               | 0.58               | 0.6364                                | 0.8559                             | 0.5563                   |
+| cosine_accuracy@5   | 0.4                | 0.8                | 0.7096                                | 0.8824                             | 0.6338                   |
+| cosine_accuracy@10  | 0.52               | 0.9                | 0.8157                                | 0.9147                             | 0.7183                   |
+| cosine_precision@1  | 0.24               | 0.28               | 0.4293                                | 0.6441                             | 0.419                    |
+| cosine_precision@3  | 0.1667             | 0.2267             | 0.2121                                | 0.2853                             | 0.2007                   |
+| cosine_precision@5  | 0.136              | 0.224              | 0.1419                                | 0.1765                             | 0.143                    |
+| cosine_precision@10 | 0.094              | 0.158              | 0.0816                                | 0.0915                             | 0.0884                   |
+| cosine_recall@1     | 0.0668             | 0.073              | 0.4293                                | 0.6441                             | 0.3594                   |
+| cosine_recall@3     | 0.1388             | 0.1727             | 0.6364                                | 0.8559                             | 0.4815                   |
+| cosine_recall@5     | 0.1837             | 0.2763             | 0.7096                                | 0.8824                             | 0.5483                   |
+| cosine_recall@10    | 0.2667             | 0.3773             | 0.8157                                | 0.9147                             | 0.6505                   |
+| **cosine_ndcg@10**  | **0.2222**         | **0.324**          | **0.6115**                            | **0.7924**                         | **0.5156**               |
+| cosine_mrr@10       | 0.3134             | 0.4852             | 0.5473                                | 0.7519                             | 0.5074                   |
+| cosine_map@100      | 0.1854             | 0.2615             | 0.5554                                | 0.7547                             | 0.4671                   |
 
 <!--
 ## Bias, Risks and Limitations
@@ -632,7 +632,7 @@ You can finetune this model on your own dataset.
   |         | anchor                                                                             | positive                                                                            | negative                                                                             |
   |:--------|:-----------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|
   | type    | string                                                                             | string                                                                              | string                                                                               |
-  | details | <ul><li>min: 11 tokens</li><li>mean: 41.76 tokens</li><li>max: 99 tokens</li></ul> | <ul><li>min: 25 tokens</li><li>mean: 131.1 tokens</li><li>max: 256 tokens</li></ul> | <ul><li>min: 14 tokens</li><li>mean: 121.07 tokens</li><li>max: 256 tokens</li></ul> |
+  | details | <ul><li>min: 11 tokens</li><li>mean: 41.76 tokens</li><li>max: 99 tokens</li></ul> | <ul><li>min: 25 tokens</li><li>mean: 119.1 tokens</li><li>max: 192 tokens</li></ul> | <ul><li>min: 14 tokens</li><li>mean: 113.91 tokens</li><li>max: 192 tokens</li></ul> |
 * Samples:
   | anchor                                                                                                             | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | negative                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
   |:-------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -653,10 +653,10 @@ You can finetune this model on your own dataset.
 * Size: 3,742 training samples
 * Columns: <code>anchor</code> and <code>positive</code>
 * Approximate statistics based on the first 1000 samples:
-  |         | anchor                                                                              | positive                                                                            |
-  |:--------|:------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
-  | type    | string                                                                              | string                                                                              |
-  | details | <ul><li>min: 13 tokens</li><li>mean: 116.0 tokens</li><li>max: 256 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 147.66 tokens</li><li>max: 256 tokens</li></ul> |
+  |         | anchor                                                                               | positive                                                                            |
+  |:--------|:-------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
+  | type    | string                                                                               | string                                                                              |
+  | details | <ul><li>min: 13 tokens</li><li>mean: 108.12 tokens</li><li>max: 192 tokens</li></ul> | <ul><li>min: 6 tokens</li><li>mean: 130.94 tokens</li><li>max: 192 tokens</li></ul> |
 * Samples:
   | anchor                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
   |:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -680,7 +680,7 @@ You can finetune this model on your own dataset.
   |         | anchor                                                                               | positive                                                                             |
   |:--------|:-------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|
   | type    | string                                                                               | string                                                                               |
-  | details | <ul><li>min: 27 tokens</li><li>mean: 161.25 tokens</li><li>max: 256 tokens</li></ul> | <ul><li>min: 15 tokens</li><li>mean: 203.26 tokens</li><li>max: 256 tokens</li></ul> |
+  | details | <ul><li>min: 27 tokens</li><li>mean: 141.93 tokens</li><li>max: 192 tokens</li></ul> | <ul><li>min: 15 tokens</li><li>mean: 166.18 tokens</li><li>max: 192 tokens</li></ul> |
 * Samples:
   | anchor                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |
   |:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -701,10 +701,10 @@ You can finetune this model on your own dataset.
 * Size: 107 training samples
 * Columns: <code>anchor</code> and <code>positive</code>
 * Approximate statistics based on the first 107 samples:
-  |         | anchor                                                                                | positive                                                                           |
-  |:--------|:--------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|
-  | type    | string                                                                                | string                                                                             |
-  | details | <ul><li>min: 107 tokens</li><li>mean: 168.68 tokens</li><li>max: 256 tokens</li></ul> | <ul><li>min: 27 tokens</li><li>mean: 82.2 tokens</li><li>max: 256 tokens</li></ul> |
+  |         | anchor                                                                                | positive                                                                            |
+  |:--------|:--------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------|
+  | type    | string                                                                                | string                                                                              |
+  | details | <ul><li>min: 107 tokens</li><li>mean: 164.29 tokens</li><li>max: 192 tokens</li></ul> | <ul><li>min: 27 tokens</li><li>mean: 78.31 tokens</li><li>max: 192 tokens</li></ul> |
 * Samples:
   | anchor                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
   |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -725,10 +725,10 @@ You can finetune this model on your own dataset.
 * Size: 11,180 training samples
 * Columns: <code>anchor</code> and <code>positive</code>
 * Approximate statistics based on the first 1000 samples:
-  |         | anchor                                                                              | positive                                                                           |
-  |:--------|:------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------|
-  | type    | string                                                                              | string                                                                             |
-  | details | <ul><li>min: 33 tokens</li><li>mean: 51.31 tokens</li><li>max: 105 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 58.93 tokens</li><li>max: 256 tokens</li></ul> |
+  |         | anchor                                                                              | positive                                                                          |
+  |:--------|:------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------|
+  | type    | string                                                                              | string                                                                            |
+  | details | <ul><li>min: 33 tokens</li><li>mean: 51.31 tokens</li><li>max: 105 tokens</li></ul> | <ul><li>min: 3 tokens</li><li>mean: 57.1 tokens</li><li>max: 192 tokens</li></ul> |
 * Samples:
   | anchor                                                                                                                                                                        | positive                                      |
   |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------|
@@ -776,7 +776,7 @@ You can finetune this model on your own dataset.
   |         | anchor                                                                               | positive                                                                             |
   |:--------|:-------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|
   | type    | string                                                                               | string                                                                               |
-  | details | <ul><li>min: 256 tokens</li><li>mean: 256.0 tokens</li><li>max: 256 tokens</li></ul> | <ul><li>min: 63 tokens</li><li>mean: 253.19 tokens</li><li>max: 256 tokens</li></ul> |
+  | details | <ul><li>min: 192 tokens</li><li>mean: 192.0 tokens</li><li>max: 192 tokens</li></ul> | <ul><li>min: 63 tokens</li><li>mean: 191.26 tokens</li><li>max: 192 tokens</li></ul> |
 * Samples:
   | anchor                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       |
   |:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -800,7 +800,7 @@ You can finetune this model on your own dataset.
   |         | anchor                                                                              | positive                                                                             |
   |:--------|:------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------|
   | type    | string                                                                              | string                                                                               |
-  | details | <ul><li>min: 12 tokens</li><li>mean: 38.68 tokens</li><li>max: 106 tokens</li></ul> | <ul><li>min: 21 tokens</li><li>mean: 114.45 tokens</li><li>max: 256 tokens</li></ul> |
+  | details | <ul><li>min: 12 tokens</li><li>mean: 38.68 tokens</li><li>max: 106 tokens</li></ul> | <ul><li>min: 21 tokens</li><li>mean: 111.75 tokens</li><li>max: 192 tokens</li></ul> |
 * Samples:
   | anchor                                                                                                                                                                                     | positive                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
   |:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
@@ -819,7 +819,7 @@ You can finetune this model on your own dataset.
 #### Non-Default Hyperparameters
 
 - `eval_strategy`: steps
-- `per_device_train_batch_size`: 256
+- `per_device_train_batch_size`: 64
 - `learning_rate`: 5e-06
 - `num_train_epochs`: 2
 - `warmup_ratio`: 0.1
@@ -833,7 +833,7 @@ You can finetune this model on your own dataset.
 - `do_predict`: False
 - `eval_strategy`: steps
 - `prediction_loss_only`: True
-- `per_device_train_batch_size`: 256
+- `per_device_train_batch_size`: 64
 - `per_device_eval_batch_size`: 8
 - `per_gpu_train_batch_size`: None
 - `per_gpu_eval_batch_size`: None
@@ -946,13 +946,25 @@ You can finetune this model on your own dataset.
 </details>
 
 ### Training Logs
-| Epoch  | Step | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
-|:------:|:----:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
-| 0      | 0    | 0.1972                            | 0.2052                            | 0.6560                                               | 0.8641                                            | 0.5900                                  |
-| 0.4717 | 100  | 0.2409                            | 0.2173                            | 0.6624                                               | 0.8766                                            | 0.6055                                  |
-| 0.9434 | 200  | 0.2489                            | 0.2207                            | 0.6553                                               | 0.8725                                            | 0.5998                                  |
-| 1.4151 | 300  | 0.2619                            | 0.2355                            | 0.6641                                               | 0.8790                                            | 0.5992                                  |
-| 1.8868 | 400  | 0.2620                            | 0.2333                            | 0.6609                                               | 0.8773                                            | 0.5960                                  |
+| Epoch  | Step | Training Loss | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
+|:------:|:----:|:-------------:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
+| 0      | 0    | -             | 0.1704                            | 0.2351                            | 0.6781                                               | 0.8793                                            | 0.5766                                  |
+| 0.1196 | 100  | -             | 0.2192                            | 0.2808                            | 0.6816                                               | 0.8857                                            | 0.6033                                  |
+| 0.2392 | 200  | -             | 0.2285                            | 0.2958                            | 0.6637                                               | 0.8878                                            | 0.6141                                  |
+| 0.3589 | 300  | -             | 0.2384                            | 0.3174                            | 0.6504                                               | 0.8820                                            | 0.6103                                  |
+| 0.4785 | 400  | -             | 0.2349                            | 0.3105                            | 0.6379                                               | 0.8626                                            | 0.5871                                  |
+| 0.5981 | 500  | 1.9344        | 0.2223                            | 0.3026                            | 0.6288                                               | 0.8476                                            | 0.5743                                  |
+| 0.7177 | 600  | -             | 0.2155                            | 0.3078                            | 0.6247                                               | 0.8277                                            | 0.5571                                  |
+| 0.8373 | 700  | -             | 0.2179                            | 0.3183                            | 0.6244                                               | 0.8389                                            | 0.5469                                  |
+| 0.9569 | 800  | -             | 0.2145                            | 0.3207                            | 0.6230                                               | 0.8368                                            | 0.5374                                  |
+| 1.0766 | 900  | -             | 0.2045                            | 0.3241                            | 0.6257                                               | 0.8331                                            | 0.5360                                  |
+| 1.1962 | 1000 | 0.9429        | 0.2162                            | 0.3450                            | 0.6145                                               | 0.8216                                            | 0.5296                                  |
+| 1.3158 | 1100 | -             | 0.2175                            | 0.3369                            | 0.6149                                               | 0.8160                                            | 0.5308                                  |
+| 1.4354 | 1200 | -             | 0.2274                            | 0.3246                            | 0.6095                                               | 0.8020                                            | 0.5262                                  |
+| 1.5550 | 1300 | -             | 0.2217                            | 0.3273                            | 0.6182                                               | 0.8030                                            | 0.5244                                  |
+| 1.6746 | 1400 | -             | 0.2186                            | 0.3226                            | 0.6145                                               | 0.7935                                            | 0.5196                                  |
+| 1.7943 | 1500 | 0.9098        | 0.2222                            | 0.3203                            | 0.6129                                               | 0.7898                                            | 0.5178                                  |
+| 1.9139 | 1600 | -             | 0.2222                            | 0.3240                            | 0.6115                                               | 0.7924                                            | 0.5156                                  |
 
 
 ### Framework Versions

config.json

@@ -1,26 +1,24 @@
 {
-  "_name_or_path": "sentence-transformers/all-MiniLM-L6-v2",
+  "_name_or_path": "sentence-transformers/all-mpnet-base-v2",
   "architectures": [
-    "BertModel"
+    "MPNetModel"
   ],
   "attention_probs_dropout_prob": 0.1,
-  "classifier_dropout": null,
-  "gradient_checkpointing": false,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 384,
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 1536,
-  "layer_norm_eps": 1e-12,
-  "max_position_embeddings": 512,
-  "model_type": "bert",
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "mpnet",
   "num_attention_heads": 12,
-  "num_hidden_layers": 6,
-  "pad_token_id": 0,
-  "position_embedding_type": "absolute",
+  "num_hidden_layers": 12,
+  "pad_token_id": 1,
+  "relative_attention_num_buckets": 32,
   "torch_dtype": "float32",
   "transformers_version": "4.45.2",
-  "type_vocab_size": 2,
-  "use_cache": true,
-  "vocab_size": 30522
+  "vocab_size": 30527
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:377c6c0f2f16b01e67bae8bf3d7ec4fa58444f4f55ca3465753fcaf6722e4bd4
-size 90864192
+oid sha256:b6d829c685c2a71831ec9bcba3b2fb9e107b46825a36f9eed6f4a5d0c91fb174
+size 437967672

sentence_bert_config.json

@@ -1,4 +1,4 @@
 {
-  "max_seq_length": 256,
+  "max_seq_length": 192,
   "do_lower_case": false
 }

special_tokens_map.json

@@ -1,27 +1,41 @@
 {
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "cls_token": {
-    "content": "[CLS]",
+    "content": "<s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
-  "mask_token": {
-    "content": "[MASK]",
+  "eos_token": {
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "pad_token": {
-    "content": "[PAD]",
+    "content": "<pad>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "sep_token": {
-    "content": "[SEP]",
+    "content": "</s>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer_config.json

@@ -1,63 +1,71 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "[PAD]",
+      "content": "<s>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "100": {
-      "content": "[UNK]",
+    "1": {
+      "content": "<pad>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "101": {
-      "content": "[CLS]",
+    "2": {
+      "content": "</s>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "102": {
-      "content": "[SEP]",
+    "3": {
+      "content": "<unk>",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "103": {
-      "content": "[MASK]",
+    "104": {
+      "content": "[UNK]",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "30526": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
+  "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
-  "cls_token": "[CLS]",
-  "do_basic_tokenize": true,
+  "cls_token": "<s>",
   "do_lower_case": true,
-  "mask_token": "[MASK]",
+  "eos_token": "</s>",
+  "mask_token": "<mask>",
   "max_length": 128,
-  "model_max_length": 256,
-  "never_split": null,
+  "model_max_length": 384,
   "pad_to_multiple_of": null,
-  "pad_token": "[PAD]",
+  "pad_token": "<pad>",
   "pad_token_type_id": 0,
   "padding_side": "right",
-  "sep_token": "[SEP]",
+  "sep_token": "</s>",
   "stride": 0,
   "strip_accents": null,
   "tokenize_chinese_chars": true,
-  "tokenizer_class": "BertTokenizer",
+  "tokenizer_class": "MPNetTokenizer",
   "truncation_side": "right",
   "truncation_strategy": "longest_first",
   "unk_token": "[UNK]"

vocab.txt

@@ -1,3 +1,7 @@
+<s>
+<pad>
+</s>
+<unk>
 [PAD]
 [unused0]
 [unused1]
@@ -30520,3 +30524,4 @@ necessitated
 ##：
 ##？
 ##～
+<mask>