model update

README.md

@@ -14,7 +14,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7540079365079365
+      value: 0.8504761904761905
   - task:
       name: Analogy Questions (SAT full)
       type: multiple-choice-qa
@@ -25,7 +25,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6844919786096256
+      value: 0.5427807486631016
   - task:
       name: Analogy Questions (SAT)
       type: multiple-choice-qa
@@ -36,7 +36,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.685459940652819
+      value: 0.5459940652818991
   - task:
       name: Analogy Questions (BATS)
       type: multiple-choice-qa
@@ -47,7 +47,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.7971095052807116
+      value: 0.6525847693162868
   - task:
       name: Analogy Questions (Google)
       type: multiple-choice-qa
@@ -58,7 +58,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.934
+      value: 0.794
   - task:
       name: Analogy Questions (U2)
       type: multiple-choice-qa
@@ -69,7 +69,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6535087719298246
+      value: 0.5175438596491229
   - task:
       name: Analogy Questions (U4)
       type: multiple-choice-qa
@@ -80,7 +80,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6504629629629629
+      value: 0.5370370370370371
   - task:
       name: Analogy Questions (ConceptNet Analogy)
       type: multiple-choice-qa
@@ -91,7 +91,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.44798657718120805
+      value: 0.27432885906040266
   - task:
       name: Analogy Questions (TREX Analogy)
       type: multiple-choice-qa
@@ -102,7 +102,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6502732240437158
+      value: 0.5519125683060109
   - task:
       name: Analogy Questions (NELL-ONE Analogy)
       type: multiple-choice-qa
@@ -113,7 +113,7 @@ model-index:
     metrics:
     - name: Accuracy
       type: accuracy
-      value: 0.6733333333333333
+      value: 0.71
   - task:
       name: Lexical Relation Classification (BLESS)
       type: classification
@@ -124,10 +124,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9243634172065692
+      value: 0.9135151423836071
     - name: F1 (macro)
       type: f1_macro
-      value: 0.9210584903501834
+      value: 0.9050919469118833
   - task:
       name: Lexical Relation Classification (CogALexV)
       type: classification
@@ -138,10 +138,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8647887323943662
+      value: 0.8145539906103286
     - name: F1 (macro)
       type: f1_macro
-      value: 0.7127446509718055
+      value: 0.5971186607822399
   - task:
       name: Lexical Relation Classification (EVALution)
       type: classification
@@ -152,10 +152,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.6895991332611051
+      value: 0.633261105092091
     - name: F1 (macro)
       type: f1_macro
-      value: 0.6739419788779412
+      value: 0.6237504036335496
   - task:
       name: Lexical Relation Classification (K&H+N)
       type: classification
@@ -166,10 +166,10 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.9527022327328372
+      value: 0.9474160116853307
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8701681183593395
+      value: 0.8607141611262328
   - task:
       name: Lexical Relation Classification (ROOT09)
       type: classification
@@ -180,34 +180,34 @@ model-index:
     metrics:
     - name: F1
       type: f1
-      value: 0.8953306173613286
+      value: 0.8831087433406457
     - name: F1 (macro)
       type: f1_macro
-      value: 0.8920689938172552
+      value: 0.879524520525773
 
 ---
 # relbert/relbert-roberta-base-nce-semeval2012-1
 
-RelBERT based on [roberta-large](https://huggingface.co/roberta-large) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
+RelBERT based on [roberta-base](https://huggingface.co/roberta-base) fine-tuned on [relbert/semeval2012_relational_similarity](https://huggingface.co/datasets/relbert/semeval2012_relational_similarity) (see the [`relbert`](https://github.com/asahi417/relbert) for more detail of fine-tuning).
 This model achieves the following results on the relation understanding tasks:
 - Analogy Question ([dataset](https://huggingface.co/datasets/relbert/analogy_questions), [full result](https://huggingface.co/relbert/relbert-roberta-base-nce-semeval2012-1/raw/main/analogy.forward.json)):
-    - Accuracy on SAT (full): 0.6844919786096256 
-    - Accuracy on SAT: 0.685459940652819
-    - Accuracy on BATS: 0.7971095052807116
-    - Accuracy on U2: 0.6535087719298246
-    - Accuracy on U4: 0.6504629629629629
-    - Accuracy on Google: 0.934
-    - Accuracy on ConceptNet Analogy: 0.44798657718120805
-    - Accuracy on T-Rex Analogy: 0.6502732240437158
-    - Accuracy on NELL-ONE Analogy: 0.6733333333333333
+    - Accuracy on SAT (full): 0.5427807486631016 
+    - Accuracy on SAT: 0.5459940652818991
+    - Accuracy on BATS: 0.6525847693162868
+    - Accuracy on U2: 0.5175438596491229
+    - Accuracy on U4: 0.5370370370370371
+    - Accuracy on Google: 0.794
+    - Accuracy on ConceptNet Analogy: 0.27432885906040266
+    - Accuracy on T-Rex Analogy: 0.5519125683060109
+    - Accuracy on NELL-ONE Analogy: 0.71
 - Lexical Relation Classification ([dataset](https://huggingface.co/datasets/relbert/lexical_relation_classification), [full result](https://huggingface.co/relbert/relbert-roberta-base-nce-semeval2012-1/raw/main/classification.json)):
-    - Micro F1 score on BLESS: 0.9243634172065692
-    - Micro F1 score on CogALexV: 0.8647887323943662
-    - Micro F1 score on EVALution: 0.6895991332611051
-    - Micro F1 score on K&H+N: 0.9527022327328372
-    - Micro F1 score on ROOT09: 0.8953306173613286
+    - Micro F1 score on BLESS: 0.9135151423836071
+    - Micro F1 score on CogALexV: 0.8145539906103286
+    - Micro F1 score on EVALution: 0.633261105092091
+    - Micro F1 score on K&H+N: 0.9474160116853307
+    - Micro F1 score on ROOT09: 0.8831087433406457
 - Relation Mapping ([dataset](https://huggingface.co/datasets/relbert/relation_mapping), [full result](https://huggingface.co/relbert/relbert-roberta-base-nce-semeval2012-1/raw/main/relation_mapping.json)):
-    - Accuracy on Relation Mapping: 0.7540079365079365 
+    - Accuracy on Relation Mapping: 0.8504761904761905 
 
 
 ### Usage
@@ -224,7 +224,7 @@ vector = model.get_embedding(['Tokyo', 'Japan'])  # shape of (n_dim, )
 
 ### Training hyperparameters
 
- - model: roberta-large
+ - model: roberta-base
  - max_length: 64
  - epoch: 10
  - batch: 32
@@ -239,7 +239,7 @@ vector = model.get_embedding(['Tokyo', 'Japan'])  # shape of (n_dim, )
  - split_valid: validation
  - loss_function: nce
  - classification_loss: False
- - loss_function_config: {'temperature': 0.05, 'num_negative': 100, 'num_positive': 10}
+ - loss_function_config: {'temperature': 0.05, 'num_negative': 400, 'num_positive': 10}
  - augment_negative_by_positive: True
 
 See the full configuration at [config file](https://huggingface.co/relbert/relbert-roberta-base-nce-semeval2012-1/raw/main/finetuning_config.json).

analogy.forward.json

@@ -1 +1 @@
-{"semeval2012_relational_similarity/validation": 0.7468354430379747, "scan/test": 0.27042079207920794, "sat_full/test": 0.6844919786096256, "sat/test": 0.685459940652819, "u2/test": 0.6535087719298246, "u4/test": 0.6504629629629629, "google/test": 0.934, "bats/test": 0.7971095052807116, "t_rex_relational_similarity/test": 0.6502732240437158, "conceptnet_relational_similarity/test": 0.44798657718120805, "nell_relational_similarity/test": 0.6733333333333333}
+{"semeval2012_relational_similarity/validation": 0.7721518987341772, "scan/test": 0.2592821782178218, "sat_full/test": 0.5427807486631016, "sat/test": 0.5459940652818991, "u2/test": 0.5175438596491229, "u4/test": 0.5370370370370371, "google/test": 0.794, "bats/test": 0.6525847693162868, "t_rex_relational_similarity/test": 0.5519125683060109, "conceptnet_relational_similarity/test": 0.27432885906040266, "nell_relational_similarity/test": 0.71}

classification.json

@@ -1 +1 @@
-{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9243634172065692, "test/f1_macro": 0.9210584903501834, "test/f1_micro": 0.9243634172065692, "test/p_macro": 0.9105248465838572, "test/p_micro": 0.9243634172065692, "test/r_macro": 0.9330081928345754, "test/r_micro": 0.9243634172065692, "test/f1/attri": 0.9138888888888889, "test/p/attri": 0.8844086021505376, "test/r/attri": 0.9454022988505747, "test/f1/coord": 0.9608621667612025, "test/p/coord": 0.9614074914869466, "test/r/coord": 0.9603174603174603, "test/f1/event": 0.8839835728952773, "test/p/event": 0.8670694864048338, "test/r/event": 0.9015706806282723, "test/f1/hyper": 0.9401709401709402, "test/p/hyper": 0.9375, "test/r/hyper": 0.9428571428571428, "test/f1/mero": 0.8910133843212236, "test/p/mero": 0.8493317132442284, "test/r/mero": 0.9369973190348525, "test/f1/random": 0.9364319890635681, "test/p/random": 0.9634317862165963, "test/r/random": 0.910904255319149}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8647887323943662, "test/f1_macro": 0.7127446509718055, "test/f1_micro": 0.8647887323943662, "test/p_macro": 0.751262043484815, "test/p_micro": 0.8647887323943662, "test/r_macro": 0.6828073632355707, "test/r_micro": 0.8647887323943662, "test/f1/ANT": 0.7856115107913668, "test/p/ANT": 0.8149253731343283, "test/r/ANT": 0.7583333333333333, "test/f1/HYPER": 0.621301775147929, "test/p/HYPER": 0.7142857142857143, "test/r/HYPER": 0.5497382198952879, "test/f1/PART_OF": 0.7196029776674939, "test/p/PART_OF": 0.8100558659217877, "test/r/PART_OF": 0.6473214285714286, "test/f1/RANDOM": 0.9361363274406752, "test/p/RANDOM": 0.9127329192546584, "test/r/RANDOM": 0.960771493952272, "test/f1/SYN": 0.5010706638115631, "test/p/SYN": 0.5043103448275862, "test/r/SYN": 0.4978723404255319}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.6895991332611051, "test/f1_macro": 0.6739419788779412, "test/f1_micro": 0.6895991332611051, "test/p_macro": 0.7007265450978665, "test/p_micro": 0.6895991332611051, "test/r_macro": 0.6565955304970702, "test/r_micro": 0.6895991332611051, "test/f1/Antonym": 0.8098765432098765, "test/p/Antonym": 0.830379746835443, "test/r/Antonym": 0.7903614457831325, "test/f1/HasA": 0.6466165413533835, "test/p/HasA": 0.6935483870967742, "test/r/HasA": 0.6056338028169014, "test/f1/HasProperty": 0.8377125193199382, "test/p/HasProperty": 0.8338461538461538, "test/r/HasProperty": 0.8416149068322981, "test/f1/IsA": 0.6430622009569378, "test/p/IsA": 0.5733788395904437, "test/r/IsA": 0.7320261437908496, "test/f1/MadeOf": 0.6357615894039734, "test/p/MadeOf": 0.7384615384615385, "test/r/MadeOf": 0.5581395348837209, "test/f1/PartOf": 0.7292418772563177, "test/p/PartOf": 0.7651515151515151, "test/r/PartOf": 0.696551724137931, "test/f1/Synonym": 0.4153225806451613, "test/p/Synonym": 0.4703196347031963, "test/r/Synonym": 0.37184115523465705}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9527022327328372, "test/f1_macro": 0.8701681183593395, "test/f1_micro": 0.9527022327328372, "test/p_macro": 0.8796338008860408, "test/p_micro": 0.9527022327328372, "test/r_macro": 0.8612759108696288, "test/r_micro": 0.9527022327328372, "test/f1/false": 0.9632255206025698, "test/p/false": 0.9596821659799882, "test/r/false": 0.9667951378594722, "test/f1/hypo": 0.9145679012345679, "test/p/hypo": 0.9420142421159715, "test/r/hypo": 0.8886756238003839, "test/f1/mero": 0.6439232409381662, "test/p/mero": 0.6593886462882096, "test/r/mero": 0.6291666666666667, "test/f1/sibl": 0.9589558106620538, "test/p/sibl": 0.9574501491599937, "test/r/sibl": 0.9604662151519925}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8953306173613287, "test/f1_macro": 0.8920689938172552, "test/f1_micro": 0.8953306173613286, "test/p_macro": 0.8973007374529117, "test/p_micro": 0.8953306173613287, "test/r_macro": 0.8883308156737318, "test/r_micro": 0.8953306173613287, "test/f1/COORD": 0.9720534629404617, "test/p/COORD": 0.963855421686747, "test/r/COORD": 0.9803921568627451, "test/f1/HYPER": 0.8044041450777202, "test/p/HYPER": 0.8448979591836735, "test/r/HYPER": 0.7676143386897404, "test/f1/RANDOM": 0.899749373433584, "test/p/RANDOM": 0.8831488314883149, "test/r/RANDOM": 0.9169859514687101}}
+{"lexical_relation_classification/BLESS": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9135151423836071, "test/f1_macro": 0.9050919469118833, "test/f1_micro": 0.9135151423836071, "test/p_macro": 0.907490718806211, "test/p_micro": 0.9135151423836071, "test/r_macro": 0.9030772446842444, "test/r_micro": 0.9135151423836071, "test/f1/attri": 0.9125090383224873, "test/p/attri": 0.9184861717612809, "test/r/attri": 0.9066091954022989, "test/f1/coord": 0.9517937219730941, "test/p/coord": 0.9412416851441242, "test/r/coord": 0.9625850340136054, "test/f1/event": 0.855072463768116, "test/p/event": 0.8454452405322416, "test/r/event": 0.8649214659685864, "test/f1/hyper": 0.9042709867452136, "test/p/hyper": 0.9331306990881459, "test/r/hyper": 0.8771428571428571, "test/f1/mero": 0.8750835003340014, "test/p/mero": 0.8721704394141145, "test/r/mero": 0.8780160857908847, "test/f1/random": 0.9318219703283881, "test/p/random": 0.9344700768973587, "test/r/random": 0.929188829787234}, "lexical_relation_classification/CogALexV": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8145539906103286, "test/f1_macro": 0.5971186607822399, "test/f1_micro": 0.8145539906103286, "test/p_macro": 0.6302906839320729, "test/p_micro": 0.8145539906103286, "test/r_macro": 0.5705454145775101, "test/r_micro": 0.8145539906103286, "test/f1/ANT": 0.6320610687022901, "test/p/ANT": 0.7016949152542373, "test/r/ANT": 0.575, "test/f1/HYPER": 0.5299145299145299, "test/p/HYPER": 0.58125, "test/r/HYPER": 0.4869109947643979, "test/f1/PART_OF": 0.6228710462287105, "test/p/PART_OF": 0.6844919786096256, "test/r/PART_OF": 0.5714285714285714, "test/f1/RANDOM": 0.9162827640984909, "test/p/RANDOM": 0.8912237330037083, "test/r/RANDOM": 0.9427917620137299, "test/f1/SYN": 0.2844638949671773, "test/p/SYN": 0.2927927927927928, "test/r/SYN": 0.2765957446808511}, "lexical_relation_classification/EVALution": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.633261105092091, "test/f1_macro": 0.6237504036335496, "test/f1_micro": 0.633261105092091, "test/p_macro": 0.6353583177650961, "test/p_micro": 0.633261105092091, "test/r_macro": 0.6182601389498605, "test/r_micro": 0.633261105092091, "test/f1/Antonym": 0.7310513447432763, "test/p/Antonym": 0.7419354838709677, "test/r/Antonym": 0.7204819277108434, "test/f1/HasA": 0.6369426751592356, "test/p/HasA": 0.5813953488372093, "test/r/HasA": 0.704225352112676, "test/f1/HasProperty": 0.7912772585669782, "test/p/HasProperty": 0.79375, "test/r/HasProperty": 0.7888198757763976, "test/f1/IsA": 0.5889698231009365, "test/p/IsA": 0.5637450199203188, "test/r/IsA": 0.616557734204793, "test/f1/MadeOf": 0.5960264900662252, "test/p/MadeOf": 0.6923076923076923, "test/r/MadeOf": 0.5232558139534884, "test/f1/PartOf": 0.6521739130434783, "test/p/PartOf": 0.6870229007633588, "test/r/PartOf": 0.6206896551724138, "test/f1/Synonym": 0.36981132075471695, "test/p/Synonym": 0.38735177865612647, "test/r/Synonym": 0.35379061371841153}, "lexical_relation_classification/K&H+N": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.9474160116853307, "test/f1_macro": 0.8607141611262328, "test/f1_micro": 0.9474160116853307, "test/p_macro": 0.8789181448983643, "test/p_micro": 0.9474160116853307, "test/r_macro": 0.8478727519113285, "test/r_micro": 0.9474160116853307, "test/f1/false": 0.9595643091614443, "test/p/false": 0.9659056773805947, "test/r/false": 0.9533056626148829, "test/f1/hypo": 0.901222953904045, "test/p/hypo": 0.8837638376383764, "test/r/hypo": 0.9193857965451055, "test/f1/mero": 0.6291079812206573, "test/p/mero": 0.7204301075268817, "test/r/mero": 0.5583333333333333, "test/f1/sibl": 0.9529614002187843, "test/p/sibl": 0.9455729570476042, "test/r/sibl": 0.9604662151519925}, "lexical_relation_classification/ROOT09": {"classifier_config": {"activation": "relu", "alpha": 0.0001, "batch_size": "auto", "beta_1": 0.9, "beta_2": 0.999, "early_stopping": false, "epsilon": 1e-08, "hidden_layer_sizes": [100], "learning_rate": "constant", "learning_rate_init": 0.001, "max_fun": 15000, "max_iter": 200, "momentum": 0.9, "n_iter_no_change": 10, "nesterovs_momentum": true, "power_t": 0.5, "random_state": 0, "shuffle": true, "solver": "adam", "tol": 0.0001, "validation_fraction": 0.1, "verbose": false, "warm_start": false}, "test/accuracy": 0.8831087433406456, "test/f1_macro": 0.879524520525773, "test/f1_micro": 0.8831087433406457, "test/p_macro": 0.8809300714710512, "test/p_micro": 0.8831087433406456, "test/r_macro": 0.8782651144622745, "test/r_micro": 0.8831087433406456, "test/f1/COORD": 0.9688073394495412, "test/p/COORD": 0.967032967032967, "test/r/COORD": 0.9705882352941176, "test/f1/HYPER": 0.7779171894604768, "test/p/HYPER": 0.7898089171974523, "test/r/HYPER": 0.7663782447466008, "test/f1/RANDOM": 0.8918490326673009, "test/p/RANDOM": 0.8859483301827347, "test/r/RANDOM": 0.8978288633461047}}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "roberta-large",
+  "_name_or_path": "roberta-base",
   "architectures": [
     "RobertaModel"
   ],
@@ -9,19 +9,19 @@
   "eos_token_id": 2,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 1024,
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 4096,
+  "intermediate_size": 3072,
   "layer_norm_eps": 1e-05,
   "max_position_embeddings": 514,
   "model_type": "roberta",
-  "num_attention_heads": 16,
-  "num_hidden_layers": 24,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
   "pad_token_id": 1,
   "position_embedding_type": "absolute",
   "relbert_config": {
     "aggregation_mode": "average_no_mask",
-    "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <subj> is the <mask> of <obj>"
+    "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <obj>  is <subj>\u2019s <mask>"
   },
   "torch_dtype": "float32",
   "transformers_version": "4.26.1",

finetuning_config.json

@@ -1,6 +1,6 @@
 {
-  "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <subj> is the <mask> of <obj>",
-  "model": "roberta-large",
+  "template": "I wasn\u2019t aware of this relationship, but I just read in the encyclopedia that <obj>  is <subj>\u2019s <mask>",
+  "model": "roberta-base",
   "max_length": 64,
   "epoch": 10,
   "batch": 32,
@@ -17,7 +17,7 @@
   "classification_loss": false,
   "loss_function_config": {
     "temperature": 0.05,
-    "num_negative": 100,
+    "num_negative": 400,
     "num_positive": 10
   },
   "augment_negative_by_positive": true

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee1a100836ccfa7ee54d098445038c13a013242ed51dcc5b426399caf0769b2d
-size 1421575277
+oid sha256:1d0fcb094b4373e4c40bf130a45b9035be8703df3360c2c9e77950680f9b3a32
+size 498652017

tokenizer.json

@@ -53,8 +53,7 @@
   "pre_tokenizer": {
     "type": "ByteLevel",
     "add_prefix_space": false,
-    "trim_offsets": true,
-    "use_regex": true
+    "trim_offsets": true
   },
   "post_processor": {
     "type": "RobertaProcessing",
@@ -72,8 +71,7 @@
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,
-    "trim_offsets": true,
-    "use_regex": true
+    "trim_offsets": true
   },
   "model": {
     "type": "BPE",

tokenizer_config.json

@@ -6,7 +6,7 @@
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 512,
-  "name_or_path": "roberta-large",
+  "name_or_path": "roberta-base",
   "pad_token": "<pad>",
   "sep_token": "</s>",
   "special_tokens_map_file": null,