(self, audio_feature, feature_len, decode_step, tf_rate=0.0, teacher=None,
                emb_decoder=None, get_dec_state=False)
return ctc_output, encode_len, att_output, att_seq, dec_state
Arguments
            audio_feature - [BxTxD] Acoustic feature with shape 
            feature_len   - [B]     Length of each sample in a batch
            decode_step   - [int]   The maximum number of attention decoder steps 
            tf_rate       - [0,1]   The probability to perform teacher forcing for each step
            teacher       - [BxL] Ground truth for teacher forcing with sentence length L
            emb_decoder   - [obj]   Introduces the word embedding decoder, different behavior for training/inference
                                    At training stage, this ONLY affects self-sampling (output remains the same)
                                    At inference stage, this affects output to become log prob. with distribution fusion
            get_dec_state - [bool]  If true, return decoder state [BxLxD] for other purpose
Arguments
            audio_feature - [BxTxD] Acoustic feature with shape 
            feature_len   - [B]     Length of each sample in a batch
            decode_step   - [int]   The maximum number of attention decoder steps 
            tf_rate       - [0,1]   The probability to perform teacher forcing for each step
            teacher       - [BxL] Ground truth for teacher forcing with sentence length L
            emb_decoder   - [obj]   Introduces the word embedding decoder, different behavior for training/inference
                                    At training stage, this ONLY affects self-sampling (output remains the same)
                                    At inference stage, this affects output to become log prob. with distribution fusion
            get_dec_state - [bool]  If true, return decoder state [BxLxD] for other purpose
Arguments audio_feature - [ BxTxD ] Acoustic feature with shape feature_len - [ B ] Length of each sample in a batch decode_step - [ int ] The maximum number of attention decoder steps tf_rate - [ 0 1 ] The probability to perform teacher forcing for each step teacher - [ BxL ] Ground truth for teacher forcing with sentence length L emb_decoder - [ obj ] Introduces the word embedding decoder different behavior for training / inference At training stage this ONLY affects self - sampling ( output remains the same ) At inference stage this affects output to become log prob . with distribution fusion get_dec_state - [ bool ] If true return decoder state [ BxLxD ] for other purpose
def forward(self, audio_feature, feature_len, decode_step, tf_rate=0.0, teacher=None,
                emb_decoder=None, get_dec_state=False):
        '''
        Arguments
            audio_feature - [BxTxD] Acoustic feature with shape 
            feature_len   - [B]     Length of each sample in a batch
            decode_step   - [int]   The maximum number of attention decoder steps 
            tf_rate       - [0,1]   The probability to perform teacher forcing for each step
            teacher       - [BxL] Ground truth for teacher forcing with sentence length L
            emb_decoder   - [obj]   Introduces the word embedding decoder, different behavior for training/inference
                                    At training stage, this ONLY affects self-sampling (output remains the same)
                                    At inference stage, this affects output to become log prob. with distribution fusion
            get_dec_state - [bool]  If true, return decoder state [BxLxD] for other purpose
        '''
        # Init
        bs = audio_feature.shape[0]
        ctc_output, att_output, att_seq = None, None, None
        dec_state = [] if get_dec_state else None

        # Encode
        encode_feature, encode_len = self.encoder(audio_feature, feature_len)

        # CTC based decoding
        if self.enable_ctc:
            ctc_output = F.log_softmax(self.ctc_layer(encode_feature), dim=-1)

        # Attention based decoding
        if self.enable_att:
            # Init (init char = <SOS>, reset all rnn state and cell)
            self.decoder.init_state(bs)
            self.attention.reset_mem()
            last_char = self.pre_embed(torch.zeros(
                (bs), dtype=torch.long, device=encode_feature.device))
            att_seq, output_seq = [], []

            # Preprocess data for teacher forcing
            if teacher is not None:
                teacher = self.embed_drop(self.pre_embed(teacher))

            # Decode
            for t in range(decode_step):
                # Attend (inputs current state of first layer, encoded features)
                attn, context = self.attention(
                    self.decoder.get_query(), encode_feature, encode_len)
                # Decode (inputs context + embedded last character)
                decoder_input = torch.cat([last_char, context], dim=-1)
                cur_char, d_state = self.decoder(decoder_input)
                # Prepare output as input of next step
                if (teacher is not None):
                    # Training stage
                    if (tf_rate == 1) or (torch.rand(1).item() <= tf_rate):
                        # teacher forcing
                        last_char = teacher[:, t, :]
                    else:
                        # self-sampling (replace by argmax may be another choice)
                        with torch.no_grad():
                            if (emb_decoder is not None) and emb_decoder.apply_fuse:
                                _, cur_prob = emb_decoder(
                                    d_state, cur_char, return_loss=False)
                            else:
                                cur_prob = cur_char.softmax(dim=-1)
                            sampled_char = Categorical(cur_prob).sample()
                        last_char = self.embed_drop(
                            self.pre_embed(sampled_char))
                else:
                    # Inference stage
                    if (emb_decoder is not None) and emb_decoder.apply_fuse:
                        _, cur_char = emb_decoder(
                            d_state, cur_char, return_loss=False)
                    # argmax for inference
                    last_char = self.pre_embed(torch.argmax(cur_char, dim=-1))

                # save output of each step
                output_seq.append(cur_char)
                att_seq.append(attn)
                if get_dec_state:
                    dec_state.append(d_state)

            att_output = torch.stack(output_seq, dim=1)  # BxTxV
            att_seq = torch.stack(att_seq, dim=2)       # BxNxDtxT
            if get_dec_state:
                dec_state = torch.stack(dec_state, dim=1)

        return ctc_output, encode_len, att_output, att_seq, dec_state
def forward ( self , audio_feature , feature_len , decode_step , tf_rate = 0.0 , teacher = None , emb_decoder = None , get_dec_state = False ) : # Init bs = audio_feature . shape [ 0 ] ctc_output , att_output , att_seq = None , None , None dec_state = [ ] if get_dec_state else None # Encode encode_feature , encode_len = self . encoder ( audio_feature , feature_len ) # CTC based decoding if self . enable_ctc : ctc_output = F . log_softmax ( self . ctc_layer ( encode_feature ) , dim = - 1 ) # Attention based decoding if self . enable_att : # Init (init char = <SOS>, reset all rnn state and cell) self . decoder . init_state ( bs ) self . attention . reset_mem ( ) last_char = self . pre_embed ( torch . zeros ( ( bs ) , dtype = torch . long , device = encode_feature . device ) ) att_seq , output_seq = [ ] , [ ] # Preprocess data for teacher forcing if teacher is not None : teacher = self . embed_drop ( self . pre_embed ( teacher ) ) # Decode for t in range ( decode_step ) : # Attend (inputs current state of first layer, encoded features) attn , context = self . attention ( self . decoder . get_query ( ) , encode_feature , encode_len ) # Decode (inputs context + embedded last character) decoder_input = torch . cat ( [ last_char , context ] , dim = - 1 ) cur_char , d_state = self . decoder ( decoder_input ) # Prepare output as input of next step if ( teacher is not None ) : # Training stage if ( tf_rate == 1 ) or ( torch . rand ( 1 ) . item ( ) <= tf_rate ) : # teacher forcing last_char = teacher [ : , t , : ] else : # self-sampling (replace by argmax may be another choice) with torch . no_grad ( ) : if ( emb_decoder is not None ) and emb_decoder . apply_fuse : _ , cur_prob = emb_decoder ( d_state , cur_char , return_loss = False ) else : cur_prob = cur_char . softmax ( dim = - 1 ) sampled_char = Categorical ( cur_prob ) . sample ( ) last_char = self . embed_drop ( self . pre_embed ( sampled_char ) ) else : # Inference stage if ( emb_decoder is not None ) and emb_decoder . apply_fuse : _ , cur_char = emb_decoder ( d_state , cur_char , return_loss = False ) # argmax for inference last_char = self . pre_embed ( torch . argmax ( cur_char , dim = - 1 ) ) # save output of each step output_seq . append ( cur_char ) att_seq . append ( attn ) if get_dec_state : dec_state . append ( d_state ) att_output = torch . stack ( output_seq , dim = 1 ) # BxTxV att_seq = torch . stack ( att_seq , dim = 2 ) # BxNxDtxT if get_dec_state : dec_state = torch . stack ( dec_state , dim = 1 ) return ctc_output , encode_len , att_output , att_seq , dec_state