(self, model, dataset, optimizer, epoch, mode='train')
Launch training procedures for one epoch

            List of the hooks:

            - train_on_start_epoch: before the training procedure for an epoch
            - train_on_start_batch: before the training precedure for a batch
            - train_on_forward: after the forward of the model
            - train_on_backward: after the backward of the loss
            - train_on_update: after the optimization step
            - train_on_print: after the print to the terminal
            - train_on_end_batch: end of the training procedure for a batch
            - train_on_end_epoch: before saving the logs in logs.json
            - train_on_flush: end of the training procedure for an epoch
Launch training procedures for one epoch
Launch training procedures for one epoch
def train_epoch(self, model, dataset, optimizer, epoch, mode='train'):
        """ Launch training procedures for one epoch

            List of the hooks:

            - train_on_start_epoch: before the training procedure for an epoch
            - train_on_start_batch: before the training precedure for a batch
            - train_on_forward: after the forward of the model
            - train_on_backward: after the backward of the loss
            - train_on_update: after the optimization step
            - train_on_print: after the print to the terminal
            - train_on_end_batch: end of the training procedure for a batch
            - train_on_end_epoch: before saving the logs in logs.json
            - train_on_flush: end of the training procedure for an epoch
        """
        utils.set_random_seed(Options()['misc']['seed'] + epoch)  # to be able to reproduce exps on reload
        Logger()('Training model on {}set for epoch {}'.format(dataset.split, epoch))
        model.train()

        timer = {
            'begin': time.time(),
            'elapsed': time.time(),
            'process': None,
            'load': None,
            'run_avg': 0
        }
        out_epoch = {}
        batch_loader = dataset.make_batch_loader()

        self.hook(f'{mode}_on_start_epoch')
        for i, batch in enumerate(batch_loader):
            timer['load'] = time.time() - timer['elapsed']
            self.hook(f'{mode}_on_start_batch')

            optimizer.zero_grad()
            out = model(batch)
            self.hook(f'{mode}_on_forward')

            if not torch.isnan(out['loss']):
                out['loss'].backward()
            else:
                Logger()('NaN detected')
            # torch.cuda.synchronize()
            self.hook(f'{mode}_on_backward')

            optimizer.step()
            # torch.cuda.synchronize()
            self.hook(f'{mode}_on_update')

            timer['process'] = time.time() - timer['elapsed']
            if i == 0:
                timer['run_avg'] = timer['process']
            else:
                timer['run_avg'] = timer['run_avg'] * 0.8 + timer['process'] * 0.2

            Logger().log_value(f'{mode}_batch.epoch', epoch, should_print=False)
            Logger().log_value(f'{mode}_batch.batch', i, should_print=False)
            Logger().log_value(f'{mode}_batch.timer.process', timer['process'], should_print=False)
            Logger().log_value(f'{mode}_batch.timer.load', timer['load'], should_print=False)

            for key, value in out.items():
                if torch.is_tensor(value):
                    if value.numel() <= 1:
                        value = value.item()  # get number from a torch scalar
                    else:
                        continue
                if isinstance(value, (list, dict, tuple)):
                    continue
                if key not in out_epoch:
                    out_epoch[key] = []
                out_epoch[key].append(value)
                Logger().log_value(f'{mode}_batch.' + key, value, should_print=False)

            if i % Options()['engine']['print_freq'] == 0 or i == len(batch_loader) - 1:
                Logger()("{}: epoch {} | batch {}/{}".format(mode, epoch, i, len(batch_loader) - 1))
                Logger()("{} elapsed: {} | left: {}".format(
                    ' ' * len(mode),
                    datetime.timedelta(seconds=math.floor(time.time() - timer['begin'])),
                    datetime.timedelta(seconds=math.floor(timer['run_avg'] * (len(batch_loader) - 1 - i)))))
                Logger()("{} process: {:.5f} | load: {:.5f}".format(' ' * len(mode), timer['process'], timer['load']))
                Logger()("{} loss: {:.5f}".format(' ' * len(mode), out['loss'].data.item()))
                self.hook(f'{mode}_on_print')

            timer['elapsed'] = time.time()
            self.hook(f'{mode}_on_end_batch')

        Logger().log_value(f'{mode}_epoch.epoch', epoch, should_print=True)
        for key, value in out_epoch.items():
            Logger().log_value(f'{mode}_epoch.' + key, np.asarray(value).mean(), should_print=True)

        self.hook(f'{mode}_on_end_epoch')
        Logger().flush()
        self.hook(f'{mode}_on_flush')
def train_epoch ( self , model , dataset , optimizer , epoch , mode = 'train' ) : utils . set_random_seed ( Options ( ) [ 'misc' ] [ 'seed' ] + epoch ) # to be able to reproduce exps on reload Logger ( ) ( 'Training model on {}set for epoch {}' . format ( dataset . split , epoch ) ) model . train ( ) timer = { 'begin' : time . time ( ) , 'elapsed' : time . time ( ) , 'process' : None , 'load' : None , 'run_avg' : 0 } out_epoch = { } batch_loader = dataset . make_batch_loader ( ) self . hook ( f'{mode}_on_start_epoch' ) for i , batch in enumerate ( batch_loader ) : timer [ 'load' ] = time . time ( ) - timer [ 'elapsed' ] self . hook ( f'{mode}_on_start_batch' ) optimizer . zero_grad ( ) out = model ( batch ) self . hook ( f'{mode}_on_forward' ) if not torch . isnan ( out [ 'loss' ] ) : out [ 'loss' ] . backward ( ) else : Logger ( ) ( 'NaN detected' ) # torch.cuda.synchronize() self . hook ( f'{mode}_on_backward' ) optimizer . step ( ) # torch.cuda.synchronize() self . hook ( f'{mode}_on_update' ) timer [ 'process' ] = time . time ( ) - timer [ 'elapsed' ] if i == 0 : timer [ 'run_avg' ] = timer [ 'process' ] else : timer [ 'run_avg' ] = timer [ 'run_avg' ] * 0.8 + timer [ 'process' ] * 0.2 Logger ( ) . log_value ( f'{mode}_batch.epoch' , epoch , should_print = False ) Logger ( ) . log_value ( f'{mode}_batch.batch' , i , should_print = False ) Logger ( ) . log_value ( f'{mode}_batch.timer.process' , timer [ 'process' ] , should_print = False ) Logger ( ) . log_value ( f'{mode}_batch.timer.load' , timer [ 'load' ] , should_print = False ) for key , value in out . items ( ) : if torch . is_tensor ( value ) : if value . numel ( ) <= 1 : value = value . item ( ) # get number from a torch scalar else : continue if isinstance ( value , ( list , dict , tuple ) ) : continue if key not in out_epoch : out_epoch [ key ] = [ ] out_epoch [ key ] . append ( value ) Logger ( ) . log_value ( f'{mode}_batch.' + key , value , should_print = False ) if i % Options ( ) [ 'engine' ] [ 'print_freq' ] == 0 or i == len ( batch_loader ) - 1 : Logger ( ) ( "{}: epoch {} | batch {}/{}" . format ( mode , epoch , i , len ( batch_loader ) - 1 ) ) Logger ( ) ( "{} elapsed: {} | left: {}" . format ( ' ' * len ( mode ) , datetime . timedelta ( seconds = math . floor ( time . time ( ) - timer [ 'begin' ] ) ) , datetime . timedelta ( seconds = math . floor ( timer [ 'run_avg' ] * ( len ( batch_loader ) - 1 - i ) ) ) ) ) Logger ( ) ( "{} process: {:.5f} | load: {:.5f}" . format ( ' ' * len ( mode ) , timer [ 'process' ] , timer [ 'load' ] ) ) Logger ( ) ( "{} loss: {:.5f}" . format ( ' ' * len ( mode ) , out [ 'loss' ] . data . item ( ) ) ) self . hook ( f'{mode}_on_print' ) timer [ 'elapsed' ] = time . time ( ) self . hook ( f'{mode}_on_end_batch' ) Logger ( ) . log_value ( f'{mode}_epoch.epoch' , epoch , should_print = True ) for key , value in out_epoch . items ( ) : Logger ( ) . log_value ( f'{mode}_epoch.' + key , np . asarray ( value ) . mean ( ) , should_print = True ) self . hook ( f'{mode}_on_end_epoch' ) Logger ( ) . flush ( ) self . hook ( f'{mode}_on_flush' )