(args, model, tokenizer)
return global_step, moving_loss.loss
Train the model
Train the model
Train the model
def train(args, model, tokenizer):
    """ Train the model """
    if xm.is_master_ordinal():
        tb_writer = SummaryWriterP(args.output_dir)

    def summary_write(*args, **kwargs):
        if xm.is_master_ordinal():
            tb_writer.add_scalar(*args, **kwargs)

    args.train_batch_size = args.per_gpu_train_batch_size #* max(1, args.n_gpu)

    train_dataloader = build_dataloader(args, tokenizer)

    if args.max_steps > 0:
        t_total = args.max_steps
        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1
    else:
        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs

    # Prepare optimizer and schedule (linear warmup and decay)
    no_decay = ['bias', 'LayerNorm.weight']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in model.named_parameters() if p.requires_grad and not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay},
        {'params': [p for n, p in model.named_parameters() if p.requires_grad and any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
    # Scale learning rate to num cores
    #args.learning_rate = args.learning_rate * xm.xrt_world_size()
    if args.sgd:
        optimizer = SGD(optimizer_grouped_parameters, lr=args.learning_rate)
    else:
        optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
    warmup_steps = args.warmup_samples // (args.train_batch_size * xm.xrt_world_size())
    if args.lr_decay:
        scheduler = WarmupLinearSchedule(optimizer, warmup_steps=warmup_steps, t_total=t_total)
    elif args.lr_cosine:
        scheduler = WarmupCosineWithHardRestartsSchedule(optimizer, warmup_steps=warmup_steps, t_total=t_total, cycles=args.num_train_epochs)
    else:
        scheduler = WarmupZeroSchedule(optimizer, warmup_steps=warmup_steps)

    # Train!
    tracker = xm.RateTracker()
    log_info("***** Running training *****")
    log_info("  Num Epochs = %d", args.num_train_epochs)
    log_info("  Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size)
    log_info("  Total train batch size (w. parallel, distributed & accumulation) = %d",
                   args.train_batch_size * args.gradient_accumulation_steps * (xm.xrt_world_size() if args.local_rank != -1 else 1))
    log_info("  Gradient Accumulation steps = %d", args.gradient_accumulation_steps)
    log_info("  Total optimization steps = %d", t_total)

    try:
        with open(os.path.join(args.model_name_or_path, 'step.txt'), 'r') as c: 
            global_step = int(c.readline())
    except OSError as e:
        global_step = 0

    moving_loss = MovingLoss(10000//args.logging_steps)

    train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=not xm.is_master_ordinal())
    try:    
        for epoch in train_iterator:
            p_train_dataloader = pl.ParallelLoader(train_dataloader, [args.device])
            epoch_iterator = tqdm(p_train_dataloader.per_device_loader(args.device), total=len(train_dataloader), desc="Iteration", disable=not xm.is_master_ordinal())

            model.train()
            for step, batch in enumerate(epoch_iterator):
                optimizer.zero_grad()
                inputs, labels = mask_tokens(batch, tokenizer, args) if args.mlm else (batch, batch)
                outputs = model(inputs, masked_lm_labels=labels) if args.mlm else model(inputs, labels=labels)
                loss = outputs[0]  # model outputs are always tuple in pytorch-transformers (see doc)

                if args.n_gpu > 1:
                    loss = loss.mean()  # mean() to average on multi-gpu parallel training
                if args.gradient_accumulation_steps > 1:
                    loss = loss / args.gradient_accumulation_steps

                loss.backward()
                if (step + 1) % args.gradient_accumulation_steps == 0:
                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)
                    xm.optimizer_step(optimizer, barrier=True)
                    scheduler.step()  
                    global_step += 1
                    tracker.add(args.train_batch_size)

                    if args.logging_steps > 0 and global_step % args.logging_steps == 0:
                        ls = loss.item() # weird. if you call loss.item() only in one process, the whole thing hangs. So call on every and log in one.
                        moving_loss.add(ls)
                        summary_write('lr', scheduler.get_last_lr()[0], global_step)
                        epoch_iterator.set_postfix(MovingLoss=f'{moving_loss.loss:.2f}', Perplexity=f'{torch.exp(torch.tensor(moving_loss.loss)):.2f}')

                    if args.save_steps > 0 and global_step % args.save_steps == 0:
                        save_state(args, model, tokenizer, global_step)
                
                if step >= 2: # TPU seems to like consistent epoch lenght
                    if xm.is_master_ordinal():
                        print(met.metrics_report())
                    exit(0)

                #    epoch_iterator.close()
                #    break
                
                if args.max_steps > 0 and step > args.max_steps:
                    epoch_iterator.close()
                    break
            
            # evaluate once in an epoch    
            if args.evaluate_during_training: 
                results = evaluate(args, model, tokenizer, f"checkpoint-{global_step}")
                log_info(f"Eval {results}")
                for key, value in results.items():
                    summary_write("eval_{}".format(key), value, global_step)
            
            # reload dataset every args.reload_data_file epochs
            if args.reload_data_file and (epoch+1) % args.reload_data_file == 0:
                train_dataloader = build_dataloader(args, tokenizer)
            
            # that's very slow on TPU
            #print_sample(model, tokenizer, args.device, args)

    except (KeyboardInterrupt, SystemExit):
        save_state(args, model, tokenizer, global_step)
        raise

    save_state(args, model, tokenizer, global_step)

    return global_step, moving_loss.loss
def train ( args , model , tokenizer ) : if xm . is_master_ordinal ( ) : tb_writer = SummaryWriterP ( args . output_dir ) def summary_write ( * args , * * kwargs ) : if xm . is_master_ordinal ( ) : tb_writer . add_scalar ( * args , * * kwargs ) args . train_batch_size = args . per_gpu_train_batch_size #* max(1, args.n_gpu) train_dataloader = build_dataloader ( args , tokenizer ) if args . max_steps > 0 : t_total = args . max_steps args . num_train_epochs = args . max_steps // ( len ( train_dataloader ) // args . gradient_accumulation_steps ) + 1 else : t_total = len ( train_dataloader ) // args . gradient_accumulation_steps * args . num_train_epochs # Prepare optimizer and schedule (linear warmup and decay) no_decay = [ 'bias' , 'LayerNorm.weight' ] optimizer_grouped_parameters = [ { 'params' : [ p for n , p in model . named_parameters ( ) if p . requires_grad and not any ( nd in n for nd in no_decay ) ] , 'weight_decay' : args . weight_decay } , { 'params' : [ p for n , p in model . named_parameters ( ) if p . requires_grad and any ( nd in n for nd in no_decay ) ] , 'weight_decay' : 0.0 } ] # Scale learning rate to num cores #args.learning_rate = args.learning_rate * xm.xrt_world_size() if args . sgd : optimizer = SGD ( optimizer_grouped_parameters , lr = args . learning_rate ) else : optimizer = AdamW ( optimizer_grouped_parameters , lr = args . learning_rate , eps = args . adam_epsilon ) warmup_steps = args . warmup_samples // ( args . train_batch_size * xm . xrt_world_size ( ) ) if args . lr_decay : scheduler = WarmupLinearSchedule ( optimizer , warmup_steps = warmup_steps , t_total = t_total ) elif args . lr_cosine : scheduler = WarmupCosineWithHardRestartsSchedule ( optimizer , warmup_steps = warmup_steps , t_total = t_total , cycles = args . num_train_epochs ) else : scheduler = WarmupZeroSchedule ( optimizer , warmup_steps = warmup_steps ) # Train! tracker = xm . RateTracker ( ) log_info ( "***** Running training *****" ) log_info ( "  Num Epochs = %d" , args . num_train_epochs ) log_info ( "  Instantaneous batch size per GPU = %d" , args . per_gpu_train_batch_size ) log_info ( "  Total train batch size (w. parallel, distributed & accumulation) = %d" , args . train_batch_size * args . gradient_accumulation_steps * ( xm . xrt_world_size ( ) if args . local_rank != - 1 else 1 ) ) log_info ( "  Gradient Accumulation steps = %d" , args . gradient_accumulation_steps ) log_info ( "  Total optimization steps = %d" , t_total ) try : with open ( os . path . join ( args . model_name_or_path , 'step.txt' ) , 'r' ) as c : global_step = int ( c . readline ( ) ) except OSError as e : global_step = 0 moving_loss = MovingLoss ( 10000 // args . logging_steps ) train_iterator = trange ( int ( args . num_train_epochs ) , desc = "Epoch" , disable = not xm . is_master_ordinal ( ) ) try : for epoch in train_iterator : p_train_dataloader = pl . ParallelLoader ( train_dataloader , [ args . device ] ) epoch_iterator = tqdm ( p_train_dataloader . per_device_loader ( args . device ) , total = len ( train_dataloader ) , desc = "Iteration" , disable = not xm . is_master_ordinal ( ) ) model . train ( ) for step , batch in enumerate ( epoch_iterator ) : optimizer . zero_grad ( ) inputs , labels = mask_tokens ( batch , tokenizer , args ) if args . mlm else ( batch , batch ) outputs = model ( inputs , masked_lm_labels = labels ) if args . mlm else model ( inputs , labels = labels ) loss = outputs [ 0 ] # model outputs are always tuple in pytorch-transformers (see doc) if args . n_gpu > 1 : loss = loss . mean ( ) # mean() to average on multi-gpu parallel training if args . gradient_accumulation_steps > 1 : loss = loss / args . gradient_accumulation_steps loss . backward ( ) if ( step + 1 ) % args . gradient_accumulation_steps == 0 : torch . nn . utils . clip_grad_norm_ ( model . parameters ( ) , args . max_grad_norm ) xm . optimizer_step ( optimizer , barrier = True ) scheduler . step ( ) global_step += 1 tracker . add ( args . train_batch_size ) if args . logging_steps > 0 and global_step % args . logging_steps == 0 : ls = loss . item ( ) # weird. if you call loss.item() only in one process, the whole thing hangs. So call on every and log in one. moving_loss . add ( ls ) summary_write ( 'lr' , scheduler . get_last_lr ( ) [ 0 ] , global_step ) epoch_iterator . set_postfix ( MovingLoss = f'{moving_loss.loss:.2f}' , Perplexity = f'{torch.exp(torch.tensor(moving_loss.loss)):.2f}' ) if args . save_steps > 0 and global_step % args . save_steps == 0 : save_state ( args , model , tokenizer , global_step ) if step >= 2 : # TPU seems to like consistent epoch lenght if xm . is_master_ordinal ( ) : print ( met . metrics_report ( ) ) exit ( 0 ) #    epoch_iterator.close() #    break if args . max_steps > 0 and step > args . max_steps : epoch_iterator . close ( ) break # evaluate once in an epoch     if args . evaluate_during_training : results = evaluate ( args , model , tokenizer , f"checkpoint-{global_step}" ) log_info ( f"Eval {results}" ) for key , value in results . items ( ) : summary_write ( "eval_{}" . format ( key ) , value , global_step ) # reload dataset every args.reload_data_file epochs if args . reload_data_file and ( epoch + 1 ) % args . reload_data_file == 0 : train_dataloader = build_dataloader ( args , tokenizer ) # that's very slow on TPU #print_sample(model, tokenizer, args.device, args) except ( KeyboardInterrupt , SystemExit ) : save_state ( args , model , tokenizer , global_step ) raise save_state ( args , model , tokenizer , global_step ) return global_step , moving_loss . loss