code: def file_generator(self,
                       filepaths,
                       max_chars_per_file=None,
                       max_chars_total=None):
      
      chars_total = 0
      for fname in filepaths:
        chars_this_file = 0
        tf.logging.info("reading file %s" % fname)
        for text in self.filepath_to_unicode_strings(fname):
          if (max_chars_per_file and
              chars_this_file + len(text) > max_chars_per_file):
            text = text[:max_chars_per_file - chars_this_file]
          if max_chars_total and chars_total + len(text) > max_chars_total:
            text = text[:max_chars_total - chars_total]
          chars_total += len(text)
          chars_this_file += len(text)
          if text:
            yield text
          if max_chars_total and chars_total >= max_chars_total:
            return
          if max_chars_per_file and chars_this_file >= max_chars_per_file:
            break
docs: Read complete text of input files and yield unicode strings.
  
      By default, one unicode string is produced per file, but this is
      not guaranteed, since subclasses can override
      filepath_to_unicode_strings().
  
      max_chars_per_file and max_chars_total can also be specified, in which
      case some strings may be truncated or dropped to limit the total
      amount of output.
  
      Args:
        filepaths: a list of strings
        max_chars_per_file: an optional integer
        max_chars_total: an optional integer
      Yields:
        unicode strings