Initial model

config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "bert_reduced_intermediate_pe_50_epochs_256_bs_5e-05_lr_0.0_warmup_VDJdb_PIRD/checkpoint-16608",
+  "architectures": [
+    "BertForMaskedLM"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 64,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 21,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.4.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 26
+}

params.json

@@ -0,0 +1,16 @@
+{
+    "datasets": [
+        "VDJdb",
+        "PIRD"
+    ],
+    "bert": "bert",
+    "config": "/home/groups/jamesz/wukevin/projects/tcr/model_configs/bert_reduced_intermediate_pe.json",
+    "outdir": "bert_reduced_intermediate_pe_50_epochs_256_bs_5e-05_lr_0.0_warmup_VDJdb_PIRD",
+    "epochs": 50,
+    "bs": 256,
+    "lr": 5e-05,
+    "warmup": 0.0,
+    "cpu": false,
+    "holdout": 0.1,
+    "noneptune": false
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e9bad29334de6caeccfa1b3295598c5612fa5fe6ad343f5c08ebca382f9bf85
+size 229641810

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "?", "sep_token": "|", "pad_token": "$", "cls_token": "*", "mask_token": "."}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": false, "do_basic_tokenize": true, "never_split": null, "unk_token": "?", "sep_token": "|", "pad_token": "$", "cls_token": "*", "mask_token": ".", "tokenize_chinese_chars": false, "strip_accents": null, "model_max_len": 45, "padding_side": "right"}

training.log

@@ -0,0 +1,25 @@
+Git commit: 53803a3acd9c7e1115233fff458d2226d7fd0c87
+PyTorch CUDA version: 10.2
+Parameter datasets: ['VDJdb', 'PIRD']
+Parameter bert: bert
+Parameter config: /home/groups/jamesz/wukevin/projects/tcr/model_configs/bert_reduced_intermediate_pe.json
+Parameter outdir: bert_reduced_intermediate_pe_50_epochs_256_bs_5e-05_lr_0.0_warmup_VDJdb_PIRD
+Parameter epochs: 50
+Parameter bs: 256
+Parameter lr: 5e-05
+Parameter warmup: 0.0
+Parameter cpu: False
+Parameter holdout: 0.1
+Parameter noneptune: False
+Filtering VDJdb species to: ['MusMusculus', 'HomoSapiens']
+VDJdb: dropping 0 entries for null cdr3 sequence
+VDJdb: dropping 0 entries for unrecognized AAs
+PIRD data TRA/TRB instances: Counter({'TRB': 46483, 'TRA': 4019, 'TRA-TRB': 637})
+PIRD data 0.1655 data labelled with antigen sequence
+PIRD: Removing 95 entires with non amino acid residues
+Creating self supervised dataset with 98225 sequences
+Maximum sequence length: 45
+Example of tokenized input: CASSQDRGPANEQFF -> [25, 9, 13, 5, 5, 8, 3, 0, 11, 12, 13, 7, 4, 8, 18, 18, 24]
+Split test with 9822 examples
+Split train with 88403 examples
+Loading vanilla BERT model

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb7d0ac4041be09c51113d2dfa606d309921c85720720f22d82983e9a11f06ea
+size 2415

vocab.txt

@@ -0,0 +1,26 @@
+R
+H
+K
+D
+E
+S
+T
+N
+Q
+C
+U
+G
+P
+A
+V
+I
+L
+M
+F
+Y
+W
+$
+.
+?
+|
+*