(doc_or_tokens: types.DocOrTokens)
return -sum(prob * math.log2(prob) for prob in probs)
Compute the entropy of words in a document.

    Args:
        doc_or_tokens: If a spaCy ``Doc``, non-punctuation tokens (words) are extracted;
            if an iterable of spaCy ``Token`` s, all non-punct elements are used.
Compute the entropy of words in a document.
Compute the entropy of words in a document .
def entropy(doc_or_tokens: types.DocOrTokens) -> float:
    """
    Compute the entropy of words in a document.

    Args:
        doc_or_tokens: If a spaCy ``Doc``, non-punctuation tokens (words) are extracted;
            if an iterable of spaCy ``Token`` s, all non-punct elements are used.
    """
    words = utils.get_words(doc_or_tokens)
    word_counts = itertoolz.frequencies(word.text for word in words)
    n_words = sum(word_counts.values())
    probs = (count / n_words for count in word_counts.values())
    return -sum(prob * math.log2(prob) for prob in probs)
def entropy ( doc_or_tokens : types . DocOrTokens ) -> float : words = utils . get_words ( doc_or_tokens ) word_counts = itertoolz . frequencies ( word . text for word in words ) n_words = sum ( word_counts . values ( ) ) probs = ( count / n_words for count in word_counts . values ( ) ) return - sum ( prob * math . log2 ( prob ) for prob in probs )