(self, input_sentences)
return batch
Data pipeline function for pre-processing.
Data pipeline function for pre-processing.
Data pipeline function for pre - processing .
def pre_process_pipeline(self, input_sentences):
    """Data pipeline function for pre-processing."""
    language = self.task_config["language"]
    clean_english = self.task_config.get("clean_english", False)
    split_by_space = self.task_config.get("split_by_space", False)
    use_word = self.task_config.get("use_word", False)

    if language == "english":
      if clean_english:
        batch = clean_english_str_tf(input_sentences)
      else:
        batch = input_sentences
    else:
      if split_by_space:
        batch = input_sentences
      else:
        if use_word:
          batch = chinese_word_cut_tf(input_sentences)
        else:
          batch = char_cut_tf(input_sentences)
    return batch
def pre_process_pipeline ( self , input_sentences ) : language = self . task_config [ "language" ] clean_english = self . task_config . get ( "clean_english" , False ) split_by_space = self . task_config . get ( "split_by_space" , False ) use_word = self . task_config . get ( "use_word" , False ) if language == "english" : if clean_english : batch = clean_english_str_tf ( input_sentences ) else : batch = input_sentences else : if split_by_space : batch = input_sentences else : if use_word : batch = chinese_word_cut_tf ( input_sentences ) else : batch = char_cut_tf ( input_sentences ) return batch