(self, pre_process_pipeline)
return all_texts, all_labels
Preparing raw data.
    For all kinds of text input, all_texts: [sentence1, ...]
    For single output, all_labels: [[label1, label2, ...]]
    For multiple outputs, all_labels: [[label1_1, ...], [label1_2, ...]]
Preparing raw data.
    For all kinds of text input, all_texts: [sentence1, ...]
    For single output, all_labels: [[label1, label2, ...]]
    For multiple outputs, all_labels: [[label1_1, ...], [label1_2, ...]]
Preparing raw data . For all kinds of text input all_texts : [ sentence1 ... ] For single output all_labels : [[ label1 label2 ... ]] For multiple outputs all_labels : [[ label1_1 ... ] [ label1_2 ... ]]
def prepare_raw_data(self, pre_process_pipeline):
    """
    Preparing raw data.
    For all kinds of text input, all_texts: [sentence1, ...]
    For single output, all_labels: [[label1, label2, ...]]
    For multiple outputs, all_labels: [[label1_1, ...], [label1_2, ...]]
    """
    if self.output_num <= 1:
      all_labels = []
    else:
      all_labels = [[] for _ in range(self.output_num)]
    all_texts = []
    for mode in self.all_modes:
      paths = self.config["data"][mode]['paths']
      paths_after_pre_process = [one_path + ".after" for one_path in paths]
      logging.debug(
          "paths_after_pre_process: {}".format(paths_after_pre_process))

      infer_without_label = bool(mode == utils.INFER and self.infer_no_label)

      for one_path, one_path_after in zip(paths, paths_after_pre_process):
        if not os.path.exists(one_path):
          raise FileNotFoundError("{} does not exist!".format(one_path))
        data_size = get_file_len([one_path])
        self.prepare_one_raw_data([one_path], one_path_after, mode,
                                  infer_without_label, pre_process_pipeline,
                                  all_texts, all_labels, data_size)
    if self.output_num <= 1:
      all_labels = [all_labels]
    return all_texts, all_labels
def prepare_raw_data ( self , pre_process_pipeline ) : if self . output_num <= 1 : all_labels = [ ] else : all_labels = [ [ ] for _ in range ( self . output_num ) ] all_texts = [ ] for mode in self . all_modes : paths = self . config [ "data" ] [ mode ] [ 'paths' ] paths_after_pre_process = [ one_path + ".after" for one_path in paths ] logging . debug ( "paths_after_pre_process: {}" . format ( paths_after_pre_process ) ) infer_without_label = bool ( mode == utils . INFER and self . infer_no_label ) for one_path , one_path_after in zip ( paths , paths_after_pre_process ) : if not os . path . exists ( one_path ) : raise FileNotFoundError ( "{} does not exist!" . format ( one_path ) ) data_size = get_file_len ( [ one_path ] ) self . prepare_one_raw_data ( [ one_path ] , one_path_after , mode , infer_without_label , pre_process_pipeline , all_texts , all_labels , data_size ) if self . output_num <= 1 : all_labels = [ all_labels ] return all_texts , all_labels