(
    doc_term_matrix: sp.csr_matrix,
    term_to_id: Dict[str, int],
    *,
    min_df: float | int = 1,
    max_df: float | int = 1.0,
    max_n_terms: Optional[int] = None,
)
return (doc_term_matrix[:, kept_indices], term_to_id)
Filter out terms that are too common and/or too rare (by document frequency),
    and compactify the top ``max_n_terms`` in the ``id_to_term`` mapping accordingly.
    Borrows heavily from the ``sklearn.feature_extraction.text`` module.

    Args:
        doc_term_matrix: M X N matrix,
            where M is the # of docs and N is the # of unique terms.
        term_to_id: Mapping of term string to unique term id,
            e.g. :attr:`Vectorizer.vocabulary_terms`.
        min_df: If float, value is the fractional proportion of the total number
            of documents and must be in [0.0, 1.0]; if int, value is the absolute number;
            filter terms whose document frequency is less than ``min_df``
        max_df: If float, value is the fractional proportion of the total number
            of documents and must be in [0.0, 1.0]; if int, value is the absolute number;
            filter terms whose document frequency is greater than ``max_df``
        max_n_terms: If specified, only include terms whose *term* frequency
            is within the top ``max_n_terms``.

    Returns:
        Sparse matrix of shape (# docs, # unique filtered terms),
        where value (i, j) is the weight of term j in doc i.

        Term to id mapping, where keys are unique *filtered* terms as strings
        and values are their corresponding integer ids.

    Raises:
        ValueError: if ``max_df`` or ``min_df`` or ``max_n_terms`` < 0.
Filter out terms that are too common and/or too rare (by document frequency),
    and compactify the top ``max_n_terms`` in the ``id_to_term`` mapping accordingly.
    Borrows heavily from the ``sklearn.feature_extraction.text`` module.
Filter out terms that are too common and / or too rare ( by document frequency ) and compactify the top max_n_terms in the id_to_term mapping accordingly . Borrows heavily from the sklearn . feature_extraction . text module .
def filter_terms_by_df(
    doc_term_matrix: sp.csr_matrix,
    term_to_id: Dict[str, int],
    *,
    min_df: float | int = 1,
    max_df: float | int = 1.0,
    max_n_terms: Optional[int] = None,
) -> Tuple[sp.csr_matrix, Dict[str, int]]:
    """
    Filter out terms that are too common and/or too rare (by document frequency),
    and compactify the top ``max_n_terms`` in the ``id_to_term`` mapping accordingly.
    Borrows heavily from the ``sklearn.feature_extraction.text`` module.

    Args:
        doc_term_matrix: M X N matrix,
            where M is the # of docs and N is the # of unique terms.
        term_to_id: Mapping of term string to unique term id,
            e.g. :attr:`Vectorizer.vocabulary_terms`.
        min_df: If float, value is the fractional proportion of the total number
            of documents and must be in [0.0, 1.0]; if int, value is the absolute number;
            filter terms whose document frequency is less than ``min_df``
        max_df: If float, value is the fractional proportion of the total number
            of documents and must be in [0.0, 1.0]; if int, value is the absolute number;
            filter terms whose document frequency is greater than ``max_df``
        max_n_terms: If specified, only include terms whose *term* frequency
            is within the top ``max_n_terms``.

    Returns:
        Sparse matrix of shape (# docs, # unique filtered terms),
        where value (i, j) is the weight of term j in doc i.

        Term to id mapping, where keys are unique *filtered* terms as strings
        and values are their corresponding integer ids.

    Raises:
        ValueError: if ``max_df`` or ``min_df`` or ``max_n_terms`` < 0.
    """
    if max_df == 1.0 and min_df == 1 and max_n_terms is None:
        return doc_term_matrix, term_to_id
    if max_df < 0 or min_df < 0 or (max_n_terms is not None and max_n_terms < 0):
        raise ValueError("max_df, min_df, and max_n_terms may not be negative")

    n_docs, n_terms = doc_term_matrix.shape
    max_doc_count = max_df if isinstance(max_df, int) else int(max_df * n_docs)
    min_doc_count = min_df if isinstance(min_df, int) else int(min_df * n_docs)
    if max_doc_count < min_doc_count:
        raise ValueError("max_df corresponds to fewer documents than min_df")

    # calculate a mask based on document frequencies
    dfs = get_doc_freqs(doc_term_matrix)
    mask = np.ones(n_terms, dtype=bool)
    if max_doc_count < n_docs:
        mask &= dfs <= max_doc_count
    if min_doc_count > 1:
        mask &= dfs >= min_doc_count
    if max_n_terms is not None and mask.sum() > max_n_terms:
        tfs = get_term_freqs(doc_term_matrix, type_="linear")
        top_mask_inds = (-tfs[mask]).argsort()[:max_n_terms]
        new_mask = np.zeros(n_terms, dtype=bool)
        new_mask[np.where(mask)[0][top_mask_inds]] = True
        mask = new_mask

    # map old term indices to new ones
    new_indices = np.cumsum(mask) - 1
    term_to_id = {
        term: new_indices[old_index]
        for term, old_index in term_to_id.items()
        if mask[old_index]
    }

    kept_indices = np.where(mask)[0]
    if len(kept_indices) == 0:
        raise ValueError(
            "After filtering, no terms remain; "
            "try a lower `min_df` or higher `max_df`"
        )

    return (doc_term_matrix[:, kept_indices], term_to_id)
def filter_terms_by_df ( doc_term_matrix : sp . csr_matrix , term_to_id : Dict [ str , int ] , * , min_df : float | int = 1 , max_df : float | int = 1.0 , max_n_terms : Optional [ int ] = None , ) -> Tuple [ sp . csr_matrix , Dict [ str , int ] ] : if max_df == 1.0 and min_df == 1 and max_n_terms is None : return doc_term_matrix , term_to_id if max_df < 0 or min_df < 0 or ( max_n_terms is not None and max_n_terms < 0 ) : raise ValueError ( "max_df, min_df, and max_n_terms may not be negative" ) n_docs , n_terms = doc_term_matrix . shape max_doc_count = max_df if isinstance ( max_df , int ) else int ( max_df * n_docs ) min_doc_count = min_df if isinstance ( min_df , int ) else int ( min_df * n_docs ) if max_doc_count < min_doc_count : raise ValueError ( "max_df corresponds to fewer documents than min_df" ) # calculate a mask based on document frequencies dfs = get_doc_freqs ( doc_term_matrix ) mask = np . ones ( n_terms , dtype = bool ) if max_doc_count < n_docs : mask &= dfs <= max_doc_count if min_doc_count > 1 : mask &= dfs >= min_doc_count if max_n_terms is not None and mask . sum ( ) > max_n_terms : tfs = get_term_freqs ( doc_term_matrix , type_ = "linear" ) top_mask_inds = ( - tfs [ mask ] ) . argsort ( ) [ : max_n_terms ] new_mask = np . zeros ( n_terms , dtype = bool ) new_mask [ np . where ( mask ) [ 0 ] [ top_mask_inds ] ] = True mask = new_mask # map old term indices to new ones new_indices = np . cumsum ( mask ) - 1 term_to_id = { term : new_indices [ old_index ] for term , old_index in term_to_id . items ( ) if mask [ old_index ] } kept_indices = np . where ( mask ) [ 0 ] if len ( kept_indices ) == 0 : raise ValueError ( "After filtering, no terms remain; " "try a lower `min_df` or higher `max_df`" ) return ( doc_term_matrix [ : , kept_indices ] , term_to_id )