code: def example_generator(self, encoder, tmp_dir, task_id):
      
      filepaths = self.text_filepaths_for_task(tmp_dir, task_id)
      if task_id >= self.num_train_shards:
        # this is dev data - limit the total length.
        max_chars_per_file = self.max_dev_chars // (
            self.num_dev_shards * len(filepaths))
      else:
        max_chars_per_file = None
      tokens = []
      for ftext in self.file_generator(
          filepaths, max_chars_per_file=max_chars_per_file):
        tokens.extend(encoder.encode(ftext))
        pos = 0
        while pos + self.sequence_length <= len(tokens):
          yield {"targets": tokens[pos:pos + self.sequence_length]}
          pos += self.sequence_length
        if pos > 0:
          tokens = tokens[pos:]
      if self.remainder_policy == "pad":
        if tokens:
          targets = tokens + [0] * (self.sequence_length - len(tokens))
          yield {"targets": targets}
      else:
        assert self.remainder_policy == "drop"
docs: Generator for examples.
  
      Args:
        encoder: a TextEncoder
        tmp_dir: a string
        task_id: an integer
      Yields:
        feature dictionaries