docs: def compile_vocab(docs, limit=1e6, verbose=0, tokenizer=Tokenizer(stem=None, lower=None, strip=None)):
      """Get the set of words used anywhere in a sequence of documents and assign an integer id
  
      This vectorizer is much faster than the scikit-learn version (and only requires low/constant RAM ?).
  
      >>> gen = ('label: ' + chr(ord('A') + i % 3)*3 for i in range(11))
      >>> d = compile_vocab(gen, verbose=0)
      >>> d
      <gensim.corpora.dictionary.Dictionary ...>
      >>> print(d)
      Dictionary(4 unique tokens: [u'AAA', u'BBB', u'CCC', u'label'])
      >>> sorted(d.token2id.values())
      [0, 1, 2, 3]
      >>> sorted(d.token2id.keys())
      [u'AAA', u'BBB', u'CCC', u'label']
      """
      tokenizer = make_tokenizer(tokenizer)
      d = Dictionary()
  
      try:
          limit = min(limit, docs.count())
          docs = docs.iterator()
      except (AttributeError, TypeError):
          pass
      for i, doc in enumerate(docs):
          # if isinstance(doc, (tuple, list)) and len(doc) == 2 and isinstance(doc[1], int):
          #     doc, score = docs
          try:
              # in case docs is a values() queryset (dicts of records in a DB table)
              doc = doc.values()
          except AttributeError:  # doc already is a values_list
              if not isinstance(doc, str):
                  doc = ' '.join([str(v) for v in doc])
              else:
                  doc = str(doc)
          if i >= limit:
              break
          d.add_documents([list(tokenizer(doc))])
          if verbose and not i % 100:
              log.info('{}: {}'.format(i, repr(d)[:120]))
      return d