(pair, vocab, indices)
return changes
Replace all occurrences of a symbol pair ('A', 'B') with a new symbol 'AB
Replace all occurrences of a symbol pair ('A', 'B') with a new symbol 'AB
Replace all occurrences of a symbol pair ( A B ) with a new symbol AB
def replace_pair(pair, vocab, indices):
    """Replace all occurrences of a symbol pair ('A', 'B') with a new symbol 'AB'"""
    first, second = pair
    pair_str = ''.join(pair)
    pair_str = pair_str.replace('\\', '\\\\')
    changes = []
    pattern = re.compile(
        r'(?<!\S)' + re.escape(first + ' ' + second) + r'(?!\S)')
    if sys.version_info < (3, 0):
        iterator = indices[pair].iteritems()
    else:
        iterator = indices[pair].items()
    for j, freq in iterator:
        if freq < 1:
            continue
        word, freq = vocab[j]
        new_word = ' '.join(word)
        new_word = pattern.sub(pair_str, new_word)
        new_word = tuple(new_word.split())

        vocab[j] = (new_word, freq)
        changes.append((j, new_word, word, freq))

    return changes
def replace_pair ( pair , vocab , indices ) : first , second = pair pair_str = '' . join ( pair ) pair_str = pair_str . replace ( '\\' , '\\\\' ) changes = [ ] pattern = re . compile ( r'(?<!\S)' + re . escape ( first + ' ' + second ) + r'(?!\S)' ) if sys . version_info < ( 3 , 0 ) : iterator = indices [ pair ] . iteritems ( ) else : iterator = indices [ pair ] . items ( ) for j , freq in iterator : if freq < 1 : continue word , freq = vocab [ j ] new_word = ' ' . join ( word ) new_word = pattern . sub ( pair_str , new_word ) new_word = tuple ( new_word . split ( ) ) vocab [ j ] = ( new_word , freq ) changes . append ( ( j , new_word , word , freq ) ) return changes