(self, one_path, one_path_after, mode,
                           infer_without_label, pre_process_pipeline, all_texts,
                           all_labels, data_size)
Prepare one raw data.
Prepare one raw data.
Prepare one raw data .
def prepare_one_raw_data(self, one_path, one_path_after, mode,
                           infer_without_label, pre_process_pipeline, all_texts,
                           all_labels, data_size):
    """Prepare one raw data."""
    text, label = self.load_a_raw_file(one_path, infer_without_label)

    batch_num = int(math.ceil(data_size / float(self.batch_size)))
    if self.multi_text:
      one_text_after = []
      for i, one_text in enumerate(text):  #to be confirmed
        one_text_iterator = get_pre_process_text_ds_iter(
            one_text, pre_process_pipeline, self.num_parallel_calls,
            self.batch_size)
        text_after_arr = self.run_dataset(one_text_iterator, batch_num)
        text_after = [one_line.decode("utf-8") for one_line in text_after_arr]
        all_texts += text_after
        one_text_after.append(text_after)
    else:
      text = text[0]
      text_iterator = get_pre_process_text_ds_iter(text, pre_process_pipeline,
                                                   self.num_parallel_calls,
                                                   self.batch_size)
      text_after_arr = self.run_dataset(text_iterator, batch_num)
      text_after = [one_line.decode("utf-8") for one_line in text_after_arr]
      all_texts += text_after
      one_text_after = text_after

    one_label_after = []
    if not infer_without_label:
      if self.multi_output:
        for i in range(self.output_num):
          label_ds = label[i].batch(self.batch_size)
          label_iterator = tf.data.make_initializable_iterator(label_ds)
          label_after_arr = self.run_dataset(label_iterator, batch_num)
          label_after_one = [
              one_line.decode("utf-8") for one_line in label_after_arr
          ]
          one_label_after.append(label_after_one)
          all_labels[i] += label_after_one
      else:
        label = label[0]
        label_ds = label.batch(self.batch_size)
        label_iterator = tf.data.make_initializable_iterator(label_ds)
        label_after_arr = self.run_dataset(label_iterator, batch_num)
        one_label_after = [
            one_line.decode("utf-8") for one_line in label_after_arr
        ]
        all_labels += one_label_after

    self.save_a_raw_file(one_label_after, one_text_after, one_path_after,
                         infer_without_label)
def prepare_one_raw_data ( self , one_path , one_path_after , mode , infer_without_label , pre_process_pipeline , all_texts , all_labels , data_size ) : text , label = self . load_a_raw_file ( one_path , infer_without_label ) batch_num = int ( math . ceil ( data_size / float ( self . batch_size ) ) ) if self . multi_text : one_text_after = [ ] for i , one_text in enumerate ( text ) : #to be confirmed one_text_iterator = get_pre_process_text_ds_iter ( one_text , pre_process_pipeline , self . num_parallel_calls , self . batch_size ) text_after_arr = self . run_dataset ( one_text_iterator , batch_num ) text_after = [ one_line . decode ( "utf-8" ) for one_line in text_after_arr ] all_texts += text_after one_text_after . append ( text_after ) else : text = text [ 0 ] text_iterator = get_pre_process_text_ds_iter ( text , pre_process_pipeline , self . num_parallel_calls , self . batch_size ) text_after_arr = self . run_dataset ( text_iterator , batch_num ) text_after = [ one_line . decode ( "utf-8" ) for one_line in text_after_arr ] all_texts += text_after one_text_after = text_after one_label_after = [ ] if not infer_without_label : if self . multi_output : for i in range ( self . output_num ) : label_ds = label [ i ] . batch ( self . batch_size ) label_iterator = tf . data . make_initializable_iterator ( label_ds ) label_after_arr = self . run_dataset ( label_iterator , batch_num ) label_after_one = [ one_line . decode ( "utf-8" ) for one_line in label_after_arr ] one_label_after . append ( label_after_one ) all_labels [ i ] += label_after_one else : label = label [ 0 ] label_ds = label . batch ( self . batch_size ) label_iterator = tf . data . make_initializable_iterator ( label_ds ) label_after_arr = self . run_dataset ( label_iterator , batch_num ) one_label_after = [ one_line . decode ( "utf-8" ) for one_line in label_after_arr ] all_labels += one_label_after self . save_a_raw_file ( one_label_after , one_text_after , one_path_after , infer_without_label )