(
    doc_term_matrix: sp.csr_matrix,
    term_to_id: Dict[str, int],
    *,
    min_ic: float = 0.0,
    max_n_terms: Optional[int] = None,
)
return (doc_term_matrix[:, kept_indices], term_to_id)
Filter out terms that are too common and/or too rare (by information content),
    and compactify the top ``max_n_terms`` in the ``id_to_term`` mapping accordingly.
    Borrows heavily from the ``sklearn.feature_extraction.text`` module.

    Args:
        doc_term_matrix: M X N sparse matrix,
            where M is the # of docs and N is the # of unique terms.
        term_to_id: Mapping of term string to unique term id,
            e.g. :attr:`Vectorizer.vocabulary_terms`.
        min_ic: Filter terms whose information content is less than this value;
            must be in [0.0, 1.0].
        max_n_terms: If specified, only include terms whose information content
            is within the top ``max_n_terms``

    Returns:
        Sparse matrix of shape (# docs, # unique filtered terms),
        where value (i, j) is the weight of term j in doc i.

        Term to id mapping, where keys are unique *filtered* terms as strings
        and values are their corresponding integer ids.

    Raises:
        ValueError: if ``min_ic`` not in [0.0, 1.0] or ``max_n_terms`` < 0.
Filter out terms that are too common and/or too rare (by information content),
    and compactify the top ``max_n_terms`` in the ``id_to_term`` mapping accordingly.
    Borrows heavily from the ``sklearn.feature_extraction.text`` module.
Filter out terms that are too common and / or too rare ( by information content ) and compactify the top max_n_terms in the id_to_term mapping accordingly . Borrows heavily from the sklearn . feature_extraction . text module .
def filter_terms_by_ic(
    doc_term_matrix: sp.csr_matrix,
    term_to_id: Dict[str, int],
    *,
    min_ic: float = 0.0,
    max_n_terms: Optional[int] = None,
) -> Tuple[sp.csr_matrix, Dict[str, int]]:
    """
    Filter out terms that are too common and/or too rare (by information content),
    and compactify the top ``max_n_terms`` in the ``id_to_term`` mapping accordingly.
    Borrows heavily from the ``sklearn.feature_extraction.text`` module.

    Args:
        doc_term_matrix: M X N sparse matrix,
            where M is the # of docs and N is the # of unique terms.
        term_to_id: Mapping of term string to unique term id,
            e.g. :attr:`Vectorizer.vocabulary_terms`.
        min_ic: Filter terms whose information content is less than this value;
            must be in [0.0, 1.0].
        max_n_terms: If specified, only include terms whose information content
            is within the top ``max_n_terms``

    Returns:
        Sparse matrix of shape (# docs, # unique filtered terms),
        where value (i, j) is the weight of term j in doc i.

        Term to id mapping, where keys are unique *filtered* terms as strings
        and values are their corresponding integer ids.

    Raises:
        ValueError: if ``min_ic`` not in [0.0, 1.0] or ``max_n_terms`` < 0.
    """
    if min_ic == 0.0 and max_n_terms is None:
        return doc_term_matrix, term_to_id
    if min_ic < 0.0 or min_ic > 1.0:
        raise ValueError("min_ic must be a float in [0.0, 1.0]")
    if max_n_terms is not None and max_n_terms < 0:
        raise ValueError("max_n_terms may not be negative")

    _, n_terms = doc_term_matrix.shape

    # calculate a mask based on document frequencies
    ics = get_information_content(doc_term_matrix)
    mask = np.ones(n_terms, dtype=bool)
    if min_ic > 0.0:
        mask &= ics >= min_ic
    if max_n_terms is not None and mask.sum() > max_n_terms:
        top_mask_inds = (-ics[mask]).argsort()[:max_n_terms]
        new_mask = np.zeros(n_terms, dtype=bool)
        new_mask[np.where(mask)[0][top_mask_inds]] = True
        mask = new_mask

    # map old term indices to new ones
    new_indices = np.cumsum(mask) - 1
    term_to_id = {
        term: new_indices[old_index]
        for term, old_index in term_to_id.items()
        if mask[old_index]
    }

    kept_indices = np.where(mask)[0]
    if len(kept_indices) == 0:
        raise ValueError("After filtering, no terms remain; try a lower `min_ic`")

    return (doc_term_matrix[:, kept_indices], term_to_id)
def filter_terms_by_ic ( doc_term_matrix : sp . csr_matrix , term_to_id : Dict [ str , int ] , * , min_ic : float = 0.0 , max_n_terms : Optional [ int ] = None , ) -> Tuple [ sp . csr_matrix , Dict [ str , int ] ] : if min_ic == 0.0 and max_n_terms is None : return doc_term_matrix , term_to_id if min_ic < 0.0 or min_ic > 1.0 : raise ValueError ( "min_ic must be a float in [0.0, 1.0]" ) if max_n_terms is not None and max_n_terms < 0 : raise ValueError ( "max_n_terms may not be negative" ) _ , n_terms = doc_term_matrix . shape # calculate a mask based on document frequencies ics = get_information_content ( doc_term_matrix ) mask = np . ones ( n_terms , dtype = bool ) if min_ic > 0.0 : mask &= ics >= min_ic if max_n_terms is not None and mask . sum ( ) > max_n_terms : top_mask_inds = ( - ics [ mask ] ) . argsort ( ) [ : max_n_terms ] new_mask = np . zeros ( n_terms , dtype = bool ) new_mask [ np . where ( mask ) [ 0 ] [ top_mask_inds ] ] = True mask = new_mask # map old term indices to new ones new_indices = np . cumsum ( mask ) - 1 term_to_id = { term : new_indices [ old_index ] for term , old_index in term_to_id . items ( ) if mask [ old_index ] } kept_indices = np . where ( mask ) [ 0 ] if len ( kept_indices ) == 0 : raise ValueError ( "After filtering, no terms remain; try a lower `min_ic`" ) return ( doc_term_matrix [ : , kept_indices ] , term_to_id )