(corpus, out_corpus, threshold=3, uncommon='UNK')
Handling of uncommon "words" (i.e. identifiers). It finds all least common identifiers (defined by threshold) and
    replaces them by 'uncommon' string.

    Parameters
    ----------
    corpus : str
        Input corpus file
    out_corpus : str
        Outfile corpus file
    threshold : int
        Number of identifier occurrences to consider it uncommon
    uncommon : str
        String to use to replace uncommon words/identifiers

    Returns
    -------
Handling of uncommon "words" (i.e. identifiers). It finds all least common identifiers (defined by threshold) and
    replaces them by 'uncommon' string.
Handling of uncommon words ( i . e . identifiers ) . It finds all least common identifiers ( defined by threshold ) and replaces them by uncommon string .
def insert_unk(corpus, out_corpus, threshold=3, uncommon='UNK'):
    """Handling of uncommon "words" (i.e. identifiers). It finds all least common identifiers (defined by threshold) and
    replaces them by 'uncommon' string.

    Parameters
    ----------
    corpus : str
        Input corpus file
    out_corpus : str
        Outfile corpus file
    threshold : int
        Number of identifier occurrences to consider it uncommon
    uncommon : str
        String to use to replace uncommon words/identifiers

    Returns
    -------
    """
    # Find least common identifiers in corpus
    f = open(corpus)
    unique = {}
    for i, x in tqdm(enumerate(_read_corpus(f)), desc='Counting identifiers in corpus'):
        for identifier in x:
            if identifier not in unique:
                unique[identifier] = 1
            else:
                unique[identifier] += 1
    n_lines = i + 1
    least_common = set([x for x in unique if unique[x] <= threshold])
    f.close()

    f = open(corpus)
    fw = open(out_corpus, mode='w')
    for line in tqdm(_read_corpus(f), total=n_lines, desc='Inserting %s' % uncommon):
        intersection = set(line) & least_common
        if len(intersection) > 0:
            new_line = []
            for item in line:
                if item in least_common:
                    new_line.append(uncommon)
                else:
                    new_line.append(item)
            fw.write(" ".join(new_line) + '\n')
        else:
            fw.write(" ".join(line) + '\n')
    f.close()
    fw.close()
def insert_unk ( corpus , out_corpus , threshold = 3 , uncommon = 'UNK' ) : # Find least common identifiers in corpus f = open ( corpus ) unique = { } for i , x in tqdm ( enumerate ( _read_corpus ( f ) ) , desc = 'Counting identifiers in corpus' ) : for identifier in x : if identifier not in unique : unique [ identifier ] = 1 else : unique [ identifier ] += 1 n_lines = i + 1 least_common = set ( [ x for x in unique if unique [ x ] <= threshold ] ) f . close ( ) f = open ( corpus ) fw = open ( out_corpus , mode = 'w' ) for line in tqdm ( _read_corpus ( f ) , total = n_lines , desc = 'Inserting %s' % uncommon ) : intersection = set ( line ) & least_common if len ( intersection ) > 0 : new_line = [ ] for item in line : if item in least_common : new_line . append ( uncommon ) else : new_line . append ( item ) fw . write ( " " . join ( new_line ) + '\n' ) else : fw . write ( " " . join ( line ) + '\n' ) f . close ( ) fw . close ( )