(self)
Training End-to-end ASR system
Training End-to-end ASR system
Training End - to - end ASR system
def exec(self):
        ''' Training End-to-end ASR system '''
        self.verbose('Total training steps {}.'.format(
            human_format(self.max_step)))
        ctc_loss, att_loss, emb_loss = None, None, None
        n_epochs = 0
        self.timer.set()

        while self.step < self.max_step:
            # Renew dataloader to enable random sampling
            if self.curriculum > 0 and n_epochs == self.curriculum:
                self.verbose(
                    'Curriculum learning ends after {} epochs, starting random sampling.'.format(n_epochs))
                self.tr_set, _, _, _, _, _ = \
                    load_dataset(self.paras.njobs, self.paras.gpu, self.paras.pin_memory,
                                 False, **self.config['data'])
            for data in self.tr_set:
                # Pre-step : update tf_rate/lr_rate and do zero_grad
                tf_rate = self.optimizer.pre_step(self.step)
                total_loss = 0

                # Fetch data
                feat, feat_len, txt, txt_len = self.fetch_data(data)
                self.timer.cnt('rd')

                # Forward model
                # Note: txt should NOT start w/ <sos>
                ctc_output, encode_len, att_output, att_align, dec_state = \
                    self.model(feat, feat_len, max(txt_len), tf_rate=tf_rate,
                               teacher=txt, get_dec_state=self.emb_reg)

                # Plugins
                if self.emb_reg:
                    emb_loss, fuse_output = self.emb_decoder(
                        dec_state, att_output, label=txt)
                    total_loss += self.emb_decoder.weight*emb_loss

                # Compute all objectives
                if ctc_output is not None:
                    if self.paras.cudnn_ctc:
                        ctc_loss = self.ctc_loss(ctc_output.transpose(0, 1),
                                                 txt.to_sparse().values().to(device='cpu', dtype=torch.int32),
                                                 [ctc_output.shape[1]] *
                                                 len(ctc_output),
                                                 txt_len.cpu().tolist())
                    else:
                        ctc_loss = self.ctc_loss(ctc_output.transpose(
                            0, 1), txt, encode_len, txt_len)
                    total_loss += ctc_loss*self.model.ctc_weight

                if att_output is not None:
                    b, t, _ = att_output.shape
                    att_output = fuse_output if self.emb_fuse else att_output
                    att_loss = self.seq_loss(
                        att_output.view(b*t, -1), txt.view(-1))
                    total_loss += att_loss*(1-self.model.ctc_weight)

                self.timer.cnt('fw')

                # Backprop
                grad_norm = self.backward(total_loss)
                self.step += 1

                # Logger
                if (self.step == 1) or (self.step % self.PROGRESS_STEP == 0):
                    self.progress('Tr stat | Loss - {:.2f} | Grad. Norm - {:.2f} | {}'
                                  .format(total_loss.cpu().item(), grad_norm, self.timer.show()))
                    self.write_log(
                        'loss', {'tr_ctc': ctc_loss, 'tr_att': att_loss})
                    self.write_log('emb_loss', {'tr': emb_loss})
                    self.write_log('wer', {'tr_att': cal_er(self.tokenizer, att_output, txt),
                                           'tr_ctc': cal_er(self.tokenizer, ctc_output, txt, ctc=True)})
                    if self.emb_fuse:
                        if self.emb_decoder.fuse_learnable:
                            self.write_log('fuse_lambda', {
                                           'emb': self.emb_decoder.get_weight()})
                        self.write_log(
                            'fuse_temp', {'temp': self.emb_decoder.get_temp()})

                # Validation
                if (self.step == 1) or (self.step % self.valid_step == 0):
                    self.validate()

                # End of step
                # https://github.com/pytorch/pytorch/issues/13246#issuecomment-529185354
                torch.cuda.empty_cache()
                self.timer.set()
                if self.step > self.max_step:
                    break
            n_epochs += 1
        self.log.close()
def exec ( self ) : self . verbose ( 'Total training steps {}.' . format ( human_format ( self . max_step ) ) ) ctc_loss , att_loss , emb_loss = None , None , None n_epochs = 0 self . timer . set ( ) while self . step < self . max_step : # Renew dataloader to enable random sampling if self . curriculum > 0 and n_epochs == self . curriculum : self . verbose ( 'Curriculum learning ends after {} epochs, starting random sampling.' . format ( n_epochs ) ) self . tr_set , _ , _ , _ , _ , _ = load_dataset ( self . paras . njobs , self . paras . gpu , self . paras . pin_memory , False , * * self . config [ 'data' ] ) for data in self . tr_set : # Pre-step : update tf_rate/lr_rate and do zero_grad tf_rate = self . optimizer . pre_step ( self . step ) total_loss = 0 # Fetch data feat , feat_len , txt , txt_len = self . fetch_data ( data ) self . timer . cnt ( 'rd' ) # Forward model # Note: txt should NOT start w/ <sos> ctc_output , encode_len , att_output , att_align , dec_state = self . model ( feat , feat_len , max ( txt_len ) , tf_rate = tf_rate , teacher = txt , get_dec_state = self . emb_reg ) # Plugins if self . emb_reg : emb_loss , fuse_output = self . emb_decoder ( dec_state , att_output , label = txt ) total_loss += self . emb_decoder . weight * emb_loss # Compute all objectives if ctc_output is not None : if self . paras . cudnn_ctc : ctc_loss = self . ctc_loss ( ctc_output . transpose ( 0 , 1 ) , txt . to_sparse ( ) . values ( ) . to ( device = 'cpu' , dtype = torch . int32 ) , [ ctc_output . shape [ 1 ] ] * len ( ctc_output ) , txt_len . cpu ( ) . tolist ( ) ) else : ctc_loss = self . ctc_loss ( ctc_output . transpose ( 0 , 1 ) , txt , encode_len , txt_len ) total_loss += ctc_loss * self . model . ctc_weight if att_output is not None : b , t , _ = att_output . shape att_output = fuse_output if self . emb_fuse else att_output att_loss = self . seq_loss ( att_output . view ( b * t , - 1 ) , txt . view ( - 1 ) ) total_loss += att_loss * ( 1 - self . model . ctc_weight ) self . timer . cnt ( 'fw' ) # Backprop grad_norm = self . backward ( total_loss ) self . step += 1 # Logger if ( self . step == 1 ) or ( self . step % self . PROGRESS_STEP == 0 ) : self . progress ( 'Tr stat | Loss - {:.2f} | Grad. Norm - {:.2f} | {}' . format ( total_loss . cpu ( ) . item ( ) , grad_norm , self . timer . show ( ) ) ) self . write_log ( 'loss' , { 'tr_ctc' : ctc_loss , 'tr_att' : att_loss } ) self . write_log ( 'emb_loss' , { 'tr' : emb_loss } ) self . write_log ( 'wer' , { 'tr_att' : cal_er ( self . tokenizer , att_output , txt ) , 'tr_ctc' : cal_er ( self . tokenizer , ctc_output , txt , ctc = True ) } ) if self . emb_fuse : if self . emb_decoder . fuse_learnable : self . write_log ( 'fuse_lambda' , { 'emb' : self . emb_decoder . get_weight ( ) } ) self . write_log ( 'fuse_temp' , { 'temp' : self . emb_decoder . get_temp ( ) } ) # Validation if ( self . step == 1 ) or ( self . step % self . valid_step == 0 ) : self . validate ( ) # End of step # https://github.com/pytorch/pytorch/issues/13246#issuecomment-529185354 torch . cuda . empty_cache ( ) self . timer . set ( ) if self . step > self . max_step : break n_epochs += 1 self . log . close ( )