(  # pylint: disable=invalid-name
    train=True, mathqa_format=False)
return drop_yield_examples
Prepares Drop inputs.

  Args:
    train: if True, then generate training examples, otherwhise generate
      validation examples (the dataset has also a test set).
    mathqa_format: if True, then floats in targets are converted to the
      the MathQA convention and wrapped in the subtract operation.
      E.g. "3.13" is converted to "subtract(const_3_13,const_0)".

  Returns:
    drop_yield_examples: a generator of Drop examples; the generator yields
    non-tokenized examples - they can be further processed using for example
    the tokenize function from this module
Prepares Drop inputs.
Prepares Drop inputs .
def CreateDropInputs(  # pylint: disable=invalid-name
    train=True, mathqa_format=False):
  """Prepares Drop inputs.

  Args:
    train: if True, then generate training examples, otherwhise generate
      validation examples (the dataset has also a test set).
    mathqa_format: if True, then floats in targets are converted to the
      the MathQA convention and wrapped in the subtract operation.
      E.g. "3.13" is converted to "subtract(const_3_13,const_0)".

  Returns:
    drop_yield_examples: a generator of Drop examples; the generator yields
    non-tokenized examples - they can be further processed using for example
    the tokenize function from this module
  """
  if train:
    dataset = tfds.load(name='drop', split='train')
  else:
    dataset = tfds.load(name='drop', split='dev')
  dataset = tfds.as_numpy(dataset)

  def drop_yield_examples(generator=None):
    del generator
    while True:
      for example in itertools.cycle(dataset):
        input_values = 'drop question: ' + example['passage'].decode(
            'utf-8') + ' ' + example['question'].decode('utf-8')
        target_values = example['answer'].decode('utf-8')
        # Apparently the dataset has some empty "target values" -
        # when such a value is encountered, the Tokenizer decides to assign
        # to it a float32 tensor and the training fails.
        if not target_values:
          continue
        if mathqa_format:
          if target_values.replace('.', '', 1).isdigit():
            target_values = convert_to_subtract(
                convert_float_to_mathqa(target_values))
        yield input_values, target_values, np.array(
            [1] * len(target_values), dtype=np.int32)

  return drop_yield_examples
def CreateDropInputs ( # pylint: disable=invalid-name train = True , mathqa_format = False ) : if train : dataset = tfds . load ( name = 'drop' , split = 'train' ) else : dataset = tfds . load ( name = 'drop' , split = 'dev' ) dataset = tfds . as_numpy ( dataset ) def drop_yield_examples ( generator = None ) : del generator while True : for example in itertools . cycle ( dataset ) : input_values = 'drop question: ' + example [ 'passage' ] . decode ( 'utf-8' ) + ' ' + example [ 'question' ] . decode ( 'utf-8' ) target_values = example [ 'answer' ] . decode ( 'utf-8' ) # Apparently the dataset has some empty "target values" - # when such a value is encountered, the Tokenizer decides to assign # to it a float32 tensor and the training fails. if not target_values : continue if mathqa_format : if target_values . replace ( '.' , '' , 1 ) . isdigit ( ) : target_values = convert_to_subtract ( convert_float_to_mathqa ( target_values ) ) yield input_values , target_values , np . array ( [ 1 ] * len ( target_values ) , dtype = np . int32 ) return drop_yield_examples