(
    candidates: List[Candidate],
    candidate_counts: Counter[str],
    topn: int,
    idf: Optional[Dict[str, float]],
)
return candidates, unique_candidates
Filter initial set of candidates to only those with sufficiently high TF or
    (if available) modified TF*IDF.
Filter initial set of candidates to only those with sufficiently high TF or
    (if available) modified TF*IDF.
Filter initial set of candidates to only those with sufficiently high TF or ( if available ) modified TF * IDF .
def _prefilter_candidates(
    candidates: List[Candidate],
    candidate_counts: Counter[str],
    topn: int,
    idf: Optional[Dict[str, float]],
) -> Tuple[List[Candidate], Set[str]]:
    """
    Filter initial set of candidates to only those with sufficiently high TF or
    (if available) modified TF*IDF.
    """
    topn_prefilter = max(3 * topn, 100)
    if idf:
        mod_tfidfs = {
            c.text: c.count * idf.get(c.text, 1) if " " not in c.text else c.count
            for c in candidates
        }
        unique_candidates = {
            ctext
            for ctext, _ in sorted(
                mod_tfidfs.items(), key=itemgetter(1), reverse=True
            )[:topn_prefilter]
        }
    else:
        unique_candidates = {
            ctext for ctext, _ in candidate_counts.most_common(topn_prefilter)
        }
    candidates = [cand for cand in candidates if cand.text in unique_candidates]
    return candidates, unique_candidates
def _prefilter_candidates ( candidates : List [ Candidate ] , candidate_counts : Counter [ str ] , topn : int , idf : Optional [ Dict [ str , float ] ] , ) -> Tuple [ List [ Candidate ] , Set [ str ] ] : topn_prefilter = max ( 3 * topn , 100 ) if idf : mod_tfidfs = { c . text : c . count * idf . get ( c . text , 1 ) if " " not in c . text else c . count for c in candidates } unique_candidates = { ctext for ctext , _ in sorted ( mod_tfidfs . items ( ) , key = itemgetter ( 1 ) , reverse = True ) [ : topn_prefilter ] } else : unique_candidates = { ctext for ctext , _ in candidate_counts . most_common ( topn_prefilter ) } candidates = [ cand for cand in candidates if cand . text in unique_candidates ] return candidates , unique_candidates