(features, seq_length, is_training, drop_remainder)
return input_fn
Creates an `input_fn` closure to be passed to TPUEstimator.
Creates an `input_fn` closure to be passed to TPUEstimator.
Creates an input_fn closure to be passed to TPUEstimator .
def input_fn_builder(features, seq_length, is_training, drop_remainder):
  """Creates an `input_fn` closure to be passed to TPUEstimator."""

  all_input_ids = []
  all_input_mask = []
  all_segment_ids = []
  all_label_ids = []

  for feature in features:
    all_input_ids.append(feature.input_ids)
    all_input_mask.append(feature.input_mask)
    all_segment_ids.append(feature.segment_ids)
    all_label_ids.append(feature.label_id)

  def input_fn(params):
    """The actual input function."""
    batch_size = params["batch_size"]

    num_examples = len(features)

    # This is for demo purposes and does NOT scale to large data sets. We do
    # not use Dataset.from_generator() because that uses tf.py_func which is
    # not TPU compatible. The right way to load data is with TFRecordReader.
    d = tf.data.Dataset.from_tensor_slices({
        "input_ids":
            tf.constant(
                all_input_ids, shape=[num_examples, seq_length],
                dtype=tf.int32),
        "input_mask":
            tf.constant(
                all_input_mask,
                shape=[num_examples, seq_length],
                dtype=tf.int32),
        "segment_ids":
            tf.constant(
                all_segment_ids,
                shape=[num_examples, seq_length],
                dtype=tf.int32),
        "label_ids":
            tf.constant(all_label_ids, shape=[num_examples], dtype=tf.int32),
    })

    if is_training:
      d = d.repeat()
      d = d.shuffle(buffer_size=100)

    d = d.batch(batch_size=batch_size, drop_remainder=drop_remainder)
    return d

  return input_fn
def input_fn_builder ( features , seq_length , is_training , drop_remainder ) : all_input_ids = [ ] all_input_mask = [ ] all_segment_ids = [ ] all_label_ids = [ ] for feature in features : all_input_ids . append ( feature . input_ids ) all_input_mask . append ( feature . input_mask ) all_segment_ids . append ( feature . segment_ids ) all_label_ids . append ( feature . label_id ) def input_fn ( params ) : """The actual input function.""" batch_size = params [ "batch_size" ] num_examples = len ( features ) # This is for demo purposes and does NOT scale to large data sets. We do # not use Dataset.from_generator() because that uses tf.py_func which is # not TPU compatible. The right way to load data is with TFRecordReader. d = tf . data . Dataset . from_tensor_slices ( { "input_ids" : tf . constant ( all_input_ids , shape = [ num_examples , seq_length ] , dtype = tf . int32 ) , "input_mask" : tf . constant ( all_input_mask , shape = [ num_examples , seq_length ] , dtype = tf . int32 ) , "segment_ids" : tf . constant ( all_segment_ids , shape = [ num_examples , seq_length ] , dtype = tf . int32 ) , "label_ids" : tf . constant ( all_label_ids , shape = [ num_examples ] , dtype = tf . int32 ) , } ) if is_training : d = d . repeat ( ) d = d . shuffle ( buffer_size = 100 ) d = d . batch ( batch_size = batch_size , drop_remainder = drop_remainder ) return d return input_fn