Upload DistillBERTSentenceLabel

config.json

@@ -2,6 +2,10 @@
   "architectures": [
     "DistillBERTSentenceLabel"
   ],
+  "auto_map": {
+    "AutoConfig": "configuration_sentence_label.DistillBERTSentenceLabelConfig",
+    "AutoModel": "modeling_sentence_label.DistillBERTSentenceLabel"
+  },
   "model_type": "distill_bert_sentence_label",
   "torch_dtype": "float32",
   "transformers_version": "4.24.0"

configuration_sentence_label.py

@@ -0,0 +1,11 @@
+from transformers import PretrainedConfig
+
+
+class DistillBERTSentenceLabelConfig(PretrainedConfig):
+    model_type = "distill_bert_sentence_label"
+
+    def __init__(
+            self,
+            **kwargs,
+    ):
+        super().__init__(**kwargs)

modeling_sentence_label.py

@@ -0,0 +1,30 @@
+
+import torch
+from transformers import DistilBertModel, PreTrainedModel
+from configuration_sentence_label import DistillBERTSentenceLabelConfig
+
+class DistillBERTSentenceLabel(PreTrainedModel):
+    config_class = DistillBERTSentenceLabelConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.l1 = DistilBertModel.from_pretrained("distilbert-base-uncased")
+        self.pre_classifier = torch.nn.Linear(768, 768)
+        self.dropout = torch.nn.Dropout(0.3)
+        self.classifier = torch.nn.Linear(768, 1)
+        # https://glassboxmedicine.com/2019/05/26/classification-sigmoid-vs-softmax/
+        # self.softmax = torch.nn.Softmax(dim=1)
+        # self.sigmoid = torch.nn.Sigmoid() # apply sigmoid on vector of 1*4
+
+
+    def forward(self, ids=None, mask=None):
+        output_1 = self.l1(input_ids=ids, attention_mask=mask)
+        hidden_state = output_1[0]
+        pooler = hidden_state[:, 0]
+        pooler = self.pre_classifier(pooler)
+        pooler = torch.nn.ReLU()(pooler)
+        pooler = self.dropout(pooler)
+        output = self.classifier(pooler)
+        # output = self.sigmoid(output)
+        # output = self.softmax(output)
+        return output