Updated model with better training and evaluation. Test and val data included as pickle files. Older Legacy files were removed to avoid confusion.

.gitattributes

@@ -1,43 +1,5 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
-pytorch_model.bin filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/b9/11/b911c3e2edc66fc22baf3a41faa6ecd12dd7a79455502a652e5001a279b7f15f filter=lfs diff=lfs merge=lfs -text
 model.safetensors filter=lfs diff=lfs merge=lfs -text
+.git/lfs/objects/c2/28/c2283afe3337976a32efe44a79e065aa6f81371ef9d77ee94f0ba681bb40bc05 filter=lfs diff=lfs merge=lfs -text
 .git/lfs/objects/36/ca/36caa36e14df19a70caed556f433229d251ce8402b77e85b5147c3188d23de80 filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/b9/59/b95969fa605d6c02eb71edd39b5268279900f24fbe5de0f2f35dafc5774c42a6 filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/6d/b2/6db2a87329f5e868fe4b536e2cb9bc2aa144ce11680bfd0a7d30b97f5205911d filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/33/3c/333cf8c7c098370c3631b0b33ba6d007a4b89e5b5f0e4b01d75bb5a200fecd27 filter=lfs diff=lfs merge=lfs -text
-.git/lfs/objects/f8/dd/f8ddd74f6db9899ea59ba31258f3d1ef2bc610c289a535c6f2a3d1a204065faa filter=lfs diff=lfs merge=lfs -text
+test_data.pickle filter=lfs diff=lfs merge=lfs -text
+val_data.pickle filter=lfs diff=lfs merge=lfs -text

Information-Retrieval_evaluation_eval_results.csv

@@ -0,0 +1,12 @@
+epoch,steps,cos_sim-Accuracy@1,cos_sim-Accuracy@3,cos_sim-Accuracy@5,cos_sim-Accuracy@10,cos_sim-Precision@1,cos_sim-Recall@1,cos_sim-Precision@3,cos_sim-Recall@3,cos_sim-Precision@5,cos_sim-Recall@5,cos_sim-Precision@10,cos_sim-Recall@10,cos_sim-MRR@10,cos_sim-NDCG@10,cos_sim-MAP@100
+0,0,0.7430910951893551,0.8863868986693961,0.9150460593654043,0.9713408393039918,0.7430910951893551,0.7430910951893551,0.29546229955646536,0.8863868986693961,0.18300921187308086,0.9150460593654043,0.09713408393039918,0.9713408393039918,0.8213185001056036,0.8575851944745084,0.8233771807496896
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909
+0,0,0.44319344933469806,0.6427840327533265,0.7236438075742068,0.827021494370522,0.44319344933469806,0.44319344933469806,0.21426134425110882,0.6427840327533265,0.14472876151484132,0.7236438075742068,0.0827021494370522,0.827021494370522,0.5663721954151841,0.628987972602642,0.5723712607178909

Information-Retrieval_evaluation_test_results.csv

@@ -0,0 +1,6 @@
+epoch,steps,cos_sim-Accuracy@1,cos_sim-Accuracy@3,cos_sim-Accuracy@5,cos_sim-Accuracy@10,cos_sim-Precision@1,cos_sim-Recall@1,cos_sim-Precision@3,cos_sim-Recall@3,cos_sim-Precision@5,cos_sim-Recall@5,cos_sim-Precision@10,cos_sim-Recall@10,cos_sim-MRR@10,cos_sim-NDCG@10,cos_sim-MAP@100
+0,0,0.8670886075949367,0.9746835443037974,0.9810126582278481,0.9978902953586498,0.8670886075949367,0.8670886075949367,0.32489451476793246,0.9746835443037974,0.19620253164556964,0.9810126582278481,0.09978902953586499,0.9978902953586498,0.9227923447860157,0.9416311386929686,0.922909550599424
+0,0,0.7953586497890295,0.9725738396624473,0.9873417721518988,1.0,0.7953586497890295,0.7953586497890295,0.32419127988748236,0.9725738396624473,0.1974683544303798,0.9873417721518988,0.1,1.0,0.8854606188466945,0.9144827706487932,0.8854606188466949
+0,0,0.7890295358649789,0.9725738396624473,0.9873417721518988,1.0,0.7890295358649789,0.7890295358649789,0.32419127988748236,0.9725738396624473,0.1974683544303798,0.9873417721518988,0.1,1.0,0.8824777308954521,0.9123013400185618,0.8824777308954525
+0,0,0.7953586497890295,0.9725738396624473,0.9873417721518988,1.0,0.7953586497890295,0.7953586497890295,0.32419127988748236,0.9725738396624473,0.1974683544303798,0.9873417721518988,0.1,1.0,0.8854313173933424,0.9144523173130932,0.8854313173933427
+0,0,0.8037974683544303,0.9662447257383966,0.9810126582278481,1.0,0.8037974683544303,0.8037974683544303,0.3220815752461321,0.9662447257383966,0.19620253164556964,0.9810126582278481,0.1,1.0,0.8887616368629027,0.9167893473515547,0.8887616368629027

README.md

@@ -10,7 +10,7 @@ tags:
 
 ---
 
-# dell-research-harvard/lt-un-data-fine-industry-en
+# {MODEL_NAME}
 
 This is a [LinkTransformer](https://linktransformer.github.io/) model. At its core this model this is a sentence transformer model [sentence-transformers](https://www.SBERT.net) model- it just wraps around the class. 
 It is designed for quick and easy record linkage (entity-matching) through the LinkTransformer package. The tasks include clustering, deduplication, linking, aggregation and more.

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "models/linkage_un_data_en_fine_industry",
+  "_name_or_path": "sentence-transformers/multi-qa-mpnet-base-dot-v1",
   "architectures": [
     "MPNetModel"
   ],

special_tokens_map.json

@@ -9,7 +9,7 @@
   "cls_token": {
     "content": "<s>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },
@@ -37,7 +37,7 @@
   "sep_token": {
     "content": "</s>",
     "lstrip": false,
-    "normalized": false,
+    "normalized": true,
     "rstrip": false,
     "single_word": false
   },

pytorch_model.bin → test_data.pickle

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36caa36e14df19a70caed556f433229d251ce8402b77e85b5147c3188d23de80
-size 438009257
+oid sha256:695abd81d511020f8ee47022621c797f0b3823e61666d2374a3a50be7783bc64
+size 53221

val_data.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51f67c116c521e3cf6e91c654a1b5ab34d9db1aba72360aef8f8f487448ed98e
+size 400808