(self, batch)
return token_ids
Data pipeline function for common process.
    This function is used both by online training and offline inference.
Data pipeline function for common process.
    This function is used both by online training and offline inference.
Data pipeline function for common process . This function is used both by online training and offline inference .
def common_process_pipeline(self, batch):
    """
    Data pipeline function for common process.
    This function is used both by online training and offline inference.
    """
    text_vocab_file_path = self.task_config['text_vocab']
    max_seq_len = self.task_config['max_seq_len']
    vocab_path = os.path.abspath(text_vocab_file_path)
    token_ids = tokenize_sentence(batch, max_seq_len, vocab_path)
    return token_ids
def common_process_pipeline ( self , batch ) : text_vocab_file_path = self . task_config [ 'text_vocab' ] max_seq_len = self . task_config [ 'max_seq_len' ] vocab_path = os . path . abspath ( text_vocab_file_path ) token_ids = tokenize_sentence ( batch , max_seq_len , vocab_path ) return token_ids