(infile, outfile, num_symbols, min_frequency=2, verbose=False, is_dict=False)
Learn num_symbols BPE operations from vocabulary, and write to outfile.
Learn num_symbols BPE operations from vocabulary, and write to outfile.
Learn num_symbols BPE operations from vocabulary and write to outfile .
def main(infile, outfile, num_symbols, min_frequency=2, verbose=False, is_dict=False):
    """Learn num_symbols BPE operations from vocabulary, and write to outfile.
    """

    # version 0.2 changes the handling of the end-of-word token ('</w>');
    # version numbering allows bckward compatibility
    outfile.write('#version: 0.2\n')

    vocab = get_vocabulary(infile, is_dict)
    vocab = dict([(tuple(x[:-1]) + (x[-1] + '</w>',), y)
                  for (x, y) in vocab.items()])
    sorted_vocab = sorted(vocab.items(), key=lambda x: x[1], reverse=True)

    stats, indices = get_pair_statistics(sorted_vocab)
    big_stats = copy.deepcopy(stats)
    # threshold is inspired by Zipfian assumption, but should only affect speed
    threshold = max(stats.values()) / 10
    for i in range(num_symbols):
        if stats:
            most_frequent = max(stats, key=lambda x: (stats[x], x))

        # we probably missed the best pair because of pruning; go back to full statistics
        if not stats or (i and stats[most_frequent] < threshold):
            prune_stats(stats, big_stats, threshold)
            stats = copy.deepcopy(big_stats)
            most_frequent = max(stats, key=lambda x: (stats[x], x))
            # threshold is inspired by Zipfian assumption, but should only affect speed
            threshold = stats[most_frequent] * i / (i + 10000.0)
            prune_stats(stats, big_stats, threshold)

        if stats[most_frequent] < min_frequency:
            sys.stderr.write(
                'no pair has frequency >= {0}. Stopping\n'.format(min_frequency))
            break

        if verbose:
            sys.stderr.write('pair {0}: {1} {2} -> {1}{2} (frequency {3})\n'.format(
                i, most_frequent[0], most_frequent[1], stats[most_frequent]))
        outfile.write('{0} {1}\n'.format(*most_frequent))
        changes = replace_pair(most_frequent, sorted_vocab, indices)
        update_pair_statistics(most_frequent, changes, stats, indices)
        stats[most_frequent] = 0
        if not i % 100:
            prune_stats(stats, big_stats, threshold)
def main ( infile , outfile , num_symbols , min_frequency = 2 , verbose = False , is_dict = False ) : # version 0.2 changes the handling of the end-of-word token ('</w>'); # version numbering allows bckward compatibility outfile . write ( '#version: 0.2\n' ) vocab = get_vocabulary ( infile , is_dict ) vocab = dict ( [ ( tuple ( x [ : - 1 ] ) + ( x [ - 1 ] + '</w>' , ) , y ) for ( x , y ) in vocab . items ( ) ] ) sorted_vocab = sorted ( vocab . items ( ) , key = lambda x : x [ 1 ] , reverse = True ) stats , indices = get_pair_statistics ( sorted_vocab ) big_stats = copy . deepcopy ( stats ) # threshold is inspired by Zipfian assumption, but should only affect speed threshold = max ( stats . values ( ) ) / 10 for i in range ( num_symbols ) : if stats : most_frequent = max ( stats , key = lambda x : ( stats [ x ] , x ) ) # we probably missed the best pair because of pruning; go back to full statistics if not stats or ( i and stats [ most_frequent ] < threshold ) : prune_stats ( stats , big_stats , threshold ) stats = copy . deepcopy ( big_stats ) most_frequent = max ( stats , key = lambda x : ( stats [ x ] , x ) ) # threshold is inspired by Zipfian assumption, but should only affect speed threshold = stats [ most_frequent ] * i / ( i + 10000.0 ) prune_stats ( stats , big_stats , threshold ) if stats [ most_frequent ] < min_frequency : sys . stderr . write ( 'no pair has frequency >= {0}. Stopping\n' . format ( min_frequency ) ) break if verbose : sys . stderr . write ( 'pair {0}: {1} {2} -> {1}{2} (frequency {3})\n' . format ( i , most_frequent [ 0 ] , most_frequent [ 1 ] , stats [ most_frequent ] ) ) outfile . write ( '{0} {1}\n' . format ( * most_frequent ) ) changes = replace_pair ( most_frequent , sorted_vocab , indices ) update_pair_statistics ( most_frequent , changes , stats , indices ) stats [ most_frequent ] = 0 if not i % 100 : prune_stats ( stats , big_stats , threshold )