(tensors, sample_size, randomize, max_lengths,
                             end_token, input_depth, output_depth,
                             control_depth, control_pad_token)
Converts to tensors and adds hierarchical padding, if needed.
Converts to tensors and adds hierarchical padding, if needed.
Converts to tensors and adds hierarchical padding if needed .
def hierarchical_pad_tensors(tensors, sample_size, randomize, max_lengths,
                             end_token, input_depth, output_depth,
                             control_depth, control_pad_token):
  """Converts to tensors and adds hierarchical padding, if needed."""
  sampled_results = data.maybe_sample_items(
      list(zip(*tensors)), sample_size, randomize)
  if sampled_results:
    unpadded_results = data.ConverterTensors(*zip(*sampled_results))
  else:
    unpadded_results = data.ConverterTensors()
  if not max_lengths:
    return unpadded_results

  # TODO(iansimon): The way control tensors are set in ConverterTensors is
  # ugly when using a hierarchical converter. Figure out how to clean this up.

  def _hierarchical_pad(input_, output, control):
    """Pad and flatten hierarchical inputs, outputs, and controls."""
    # Pad empty segments with end tokens and flatten hierarchy.
    input_ = tf.nest.flatten(
        pad_with_element(input_, max_lengths[:-1],
                         data.np_onehot([end_token], input_depth)))
    output = tf.nest.flatten(
        pad_with_element(output, max_lengths[:-1],
                         data.np_onehot([end_token], output_depth)))
    length = np.squeeze(np.array([len(x) for x in input_], np.int32))

    # Pad and concatenate flatten hierarchy.
    input_ = np.concatenate(
        [pad_with_value(x, max_lengths[-1], 0) for x in input_])
    output = np.concatenate(
        [pad_with_value(x, max_lengths[-1], 0) for x in output])

    if np.size(control):
      control = tf.nest.flatten(
          pad_with_element(control, max_lengths[:-1],
                           data.np_onehot([control_pad_token], control_depth)))
      control = np.concatenate(
          [pad_with_value(x, max_lengths[-1], 0) for x in control])

    return input_, output, control, length

  padded_results = []
  for i, o, c, _ in zip(*unpadded_results):
    try:
      padded_results.append(_hierarchical_pad(i, o, c))
    except TooLongError:
      continue

  if padded_results:
    return data.ConverterTensors(*zip(*padded_results))
  else:
    return data.ConverterTensors()
def hierarchical_pad_tensors ( tensors , sample_size , randomize , max_lengths , end_token , input_depth , output_depth , control_depth , control_pad_token ) : sampled_results = data . maybe_sample_items ( list ( zip ( * tensors ) ) , sample_size , randomize ) if sampled_results : unpadded_results = data . ConverterTensors ( * zip ( * sampled_results ) ) else : unpadded_results = data . ConverterTensors ( ) if not max_lengths : return unpadded_results # TODO(iansimon): The way control tensors are set in ConverterTensors is # ugly when using a hierarchical converter. Figure out how to clean this up. def _hierarchical_pad ( input_ , output , control ) : """Pad and flatten hierarchical inputs, outputs, and controls.""" # Pad empty segments with end tokens and flatten hierarchy. input_ = tf . nest . flatten ( pad_with_element ( input_ , max_lengths [ : - 1 ] , data . np_onehot ( [ end_token ] , input_depth ) ) ) output = tf . nest . flatten ( pad_with_element ( output , max_lengths [ : - 1 ] , data . np_onehot ( [ end_token ] , output_depth ) ) ) length = np . squeeze ( np . array ( [ len ( x ) for x in input_ ] , np . int32 ) ) # Pad and concatenate flatten hierarchy. input_ = np . concatenate ( [ pad_with_value ( x , max_lengths [ - 1 ] , 0 ) for x in input_ ] ) output = np . concatenate ( [ pad_with_value ( x , max_lengths [ - 1 ] , 0 ) for x in output ] ) if np . size ( control ) : control = tf . nest . flatten ( pad_with_element ( control , max_lengths [ : - 1 ] , data . np_onehot ( [ control_pad_token ] , control_depth ) ) ) control = np . concatenate ( [ pad_with_value ( x , max_lengths [ - 1 ] , 0 ) for x in control ] ) return input_ , output , control , length padded_results = [ ] for i , o , c , _ in zip ( * unpadded_results ) : try : padded_results . append ( _hierarchical_pad ( i , o , c ) ) except TooLongError : continue if padded_results : return data . ConverterTensors ( * zip ( * padded_results ) ) else : return data . ConverterTensors ( )