pre-train_BERT for a specific corpus
pre-train bertHIi am trying to pre_train bert in my own corpusbut got this errors!pip install accelerate -U
!pip install transformers
import  torchvision, torchaudio
from accelerate import Accelerator
from transformers import  DataCollatorForLanguageModeling,TextDataset,TrainingArguments

from transformers import BertTokenizer, BertForPreTraining

tokenizer = BertTokenizer.from_pretrained('Jaafer/code-search-net-tokenizer')

model = BertForPreTraining.from_pretrained('bert-base-uncased')
!pip install accelerate -U
!pip install transformers
import  torchvision, torchaudio
from accelerate import Accelerator
from transformers import  DataCollatorForLanguageModeling,TextDataset,TrainingArguments

from transformers import BertTokenizer, BertForPreTraining

tokenizer = BertTokenizer.from_pretrained('Jaafer/code-search-net-tokenizer')

model = BertForPreTraining.from_pretrained('bert-base-uncased')
%%time
data="/kaggle/input/ontology/ontology.txt"

from transformers import LineByLineTextDataset

dataset = LineByLineTextDataset(
    tokenizer=tokenizer,
    file_path=data,
    block_size=128,
)
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=True, mlm_probability=0.15)
model.to(device)
from transformers import Trainer, TrainingArguments

training_args = TrainingArguments(
    output_dir="./EsperBERTo",
    overwrite_output_dir=True,
    num_train_epochs=1,
    per_gpu_train_batch_size=64,
    save_steps=10_000,
    save_total_limit=2,
    prediction_loss_only=True,
)

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=dataset,
)
trainer.train()er817×228 16.8 KB