(predict_seq_list=None, target_seq_list=None, eos_id=None)
return errs
computing token error
    :param predict_seq_list: (shape=[B,T], type=int), the list of predict token sequence
    :param target_seq_list: ([B,T], type=int), the list of target token sequence
    :param eos_id: (type=int), the end symbol of the target token sequence
    return: (type=float), the token error between predict and target token sequences
computing token error
    :param predict_seq_list: (shape=[B,T], type=int), the list of predict token sequence
    :param target_seq_list: ([B,T], type=int), the list of target token sequence
    :param eos_id: (type=int), the end symbol of the target token sequence
    return: (type=float), the token error between predict and target token sequences
computing token error : param predict_seq_list : ( shape = [ B T ] type = int ) the list of predict token sequence : param target_seq_list : ( [ B T ] type = int ) the list of target token sequence : param eos_id : ( type = int ) the end symbol of the target token sequence return : ( type = float ) the token error between predict and target token sequences
def token_error(predict_seq_list=None, target_seq_list=None, eos_id=None):
  ''' computing token error
    :param predict_seq_list: (shape=[B,T], type=int), the list of predict token sequence
    :param target_seq_list: ([B,T], type=int), the list of target token sequence
    :param eos_id: (type=int), the end symbol of the target token sequence
    return: (type=float), the token error between predict and target token sequences
    '''
  if len(predict_seq_list) != len(target_seq_list):
    raise ValueError(
        'the number of prdict sequence and target sequence is not equal!')
  if eos_id is None:
    raise ValueError('the end symbol of target sequence is None!')

  levenshtein_distance, target_length = 0.0, 0.0

  for index, cur_predict_seq in enumerate(predict_seq_list):
    cur_target_seq = target_seq_list[index]
    if eos_id in cur_target_seq:
      target_seq_end = cur_target_seq.index(eos_id)
      cur_target_seq = cur_target_seq[:target_seq_end]
    if eos_id in cur_predict_seq:
      predict_seq_end = cur_predict_seq.index(eos_id)
      cur_predict_seq = cur_predict_seq[:predict_seq_end]

    levenshtein_distance += levenshtein(cur_predict_seq, cur_target_seq)
    target_length += len(cur_target_seq)

  errs = levenshtein_distance / target_length
  return errs
def token_error ( predict_seq_list = None , target_seq_list = None , eos_id = None ) : if len ( predict_seq_list ) != len ( target_seq_list ) : raise ValueError ( 'the number of prdict sequence and target sequence is not equal!' ) if eos_id is None : raise ValueError ( 'the end symbol of target sequence is None!' ) levenshtein_distance , target_length = 0.0 , 0.0 for index , cur_predict_seq in enumerate ( predict_seq_list ) : cur_target_seq = target_seq_list [ index ] if eos_id in cur_target_seq : target_seq_end = cur_target_seq . index ( eos_id ) cur_target_seq = cur_target_seq [ : target_seq_end ] if eos_id in cur_predict_seq : predict_seq_end = cur_predict_seq . index ( eos_id ) cur_predict_seq = cur_predict_seq [ : predict_seq_end ] levenshtein_distance += levenshtein ( cur_predict_seq , cur_target_seq ) target_length += len ( cur_target_seq ) errs = levenshtein_distance / target_length return errs