code: def get_or_generate_vocabulary(data_dir,
                                 tmp_dir,
                                 data_prefix,
                                 max_page_size_exp,
                                 approx_vocab_size=32768,
                                 strip=True):
    
    num_pages_for_vocab_generation = approx_vocab_size // 3
    vocab_file = vocab_filename(approx_vocab_size, strip)
  
    def my_generator(data_prefix):
      """Line generator for vocab."""
      count = 0
      for page in corpus_page_generator(
          all_corpus_files(data_prefix)[::-1], tmp_dir, max_page_size_exp):
        revisions = page["revisions"]
        if revisions:
          text = get_text(revisions[-1], strip=strip)
          yield text
          count += 1
          if count % 100 == 0:
            tf.logging.info("reading pages for vocab %d" % count)
          if count > num_pages_for_vocab_generation:
            break
  
    return generator_utils.get_or_generate_vocab_inner(data_dir, vocab_file,
                                                       approx_vocab_size,
                                                       my_generator(data_prefix))
docs: Get or generate the vocabulary.
  
    Args:
      data_dir: a string
      tmp_dir: a string
      data_prefix: a string
      max_page_size_exp: an integer
      approx_vocab_size: an integer
      strip: a boolean
  
    Returns:
      a TextEncoder