docs: def tokenize(ngrams, min_tf=2, min_df=2, min_len=3, apply_stoplist=False):
      """
      Builds a vocabulary, and replaces words with vocab indices.
  
      Parameters
      ----------
      ngrams : dict
          Keys are paper DOIs, values are lists of (Ngram, frequency) tuples.
      apply_stoplist : bool
          If True, will exclude all N-grams that contain words in the NLTK
          stoplist.
  
      Returns
      -------
      t_ngrams : dict
          Tokenized ngrams, as doi:{i:count}.
      vocab : dict
          Vocabulary as i:term.
      token_tf : :class:`.Counter`
          Term counts for corpus, as i:count.
      """
  
      vocab = {}
      vocab_ = {}
      word_tf = Counter()
      word_df = Counter()
      token_tf = Counter()
      token_df = Counter()
      t_ngrams = {}
  
      # Get global word counts, first.
      for grams in ngrams.values():
          for g,c in grams:
              word_tf[g] += c
              word_df[g] += 1
  
      if apply_stoplist:
          stoplist = stopwords.words()
  
      # Now tokenize.
      for doi, grams in ngrams.iteritems():
          t_ngrams[doi] = []
          for g,c in grams:
              ignore = False
  
              # Ignore extremely rare words (probably garbage).
              if word_tf[g] < min_tf or word_df[g] < min_df or len(g) < min_len:
                  ignore = True
  
              # Stoplist.
              elif apply_stoplist:
                  for w in g.split():
                      if w in stoplist:
                          ignore = True
  
              if not ignore:
  
                  # Coerce unicode to string.
                  if type(g) is str:
                      g = unicode(g)
                  g = unidecode(g)
  
                  if g not in vocab.values():
                      i = len(vocab)
                      vocab[i] = g
                      vocab_[g] = i
                  else:
                      i = vocab_[g]
                  token_tf[i] += c
                  token_df[i] += 1
  
                  t_ngrams[doi].append( (i,c) )
  
      return t_ngrams, vocab, token_tf