(orig, bpe_codes, bpe_codes_reverse, vocab, separator, version, cache, glossaries=None)
return word
Encode word based on list of BPE merge operations, which are applied consecutively
Encode word based on list of BPE merge operations, which are applied consecutively
Encode word based on list of BPE merge operations which are applied consecutively
def encode(orig, bpe_codes, bpe_codes_reverse, vocab, separator, version, cache, glossaries=None):
    """Encode word based on list of BPE merge operations, which are applied consecutively
    """

    if orig in cache:
        return cache[orig]

    if orig in glossaries:
        cache[orig] = (orig,)
        return (orig,)

    if version == (0, 1):
        word = tuple(orig) + ('</w>',)
    elif version == (0, 2):  # more consistent handling of word-final segments
        word = tuple(orig[:-1]) + (orig[-1] + '</w>',)
    else:
        raise NotImplementedError

    pairs = get_pairs(word)

    if not pairs:
        return orig

    while True:
        bigram = min(pairs, key=lambda pair: bpe_codes.get(pair, float('inf')))
        if bigram not in bpe_codes:
            break
        first, second = bigram
        new_word = []
        i = 0
        while i < len(word):
            try:
                j = word.index(first, i)
                new_word.extend(word[i:j])
                i = j
            except:
                new_word.extend(word[i:])
                break

            if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
                new_word.append(first + second)
                i += 2
            else:
                new_word.append(word[i])
                i += 1
        new_word = tuple(new_word)
        word = new_word
        if len(word) == 1:
            break
        else:
            pairs = get_pairs(word)

    # don't print end-of-word symbols
    if word[-1] == '</w>':
        word = word[:-1]
    elif word[-1].endswith('</w>'):
        word = word[:-1] + (word[-1].replace('</w>', ''),)

    if vocab:
        word = check_vocab_and_split(word, bpe_codes_reverse, vocab, separator)

    cache[orig] = word
    return word
def encode ( orig , bpe_codes , bpe_codes_reverse , vocab , separator , version , cache , glossaries = None ) : if orig in cache : return cache [ orig ] if orig in glossaries : cache [ orig ] = ( orig , ) return ( orig , ) if version == ( 0 , 1 ) : word = tuple ( orig ) + ( '</w>' , ) elif version == ( 0 , 2 ) : # more consistent handling of word-final segments word = tuple ( orig [ : - 1 ] ) + ( orig [ - 1 ] + '</w>' , ) else : raise NotImplementedError pairs = get_pairs ( word ) if not pairs : return orig while True : bigram = min ( pairs , key = lambda pair : bpe_codes . get ( pair , float ( 'inf' ) ) ) if bigram not in bpe_codes : break first , second = bigram new_word = [ ] i = 0 while i < len ( word ) : try : j = word . index ( first , i ) new_word . extend ( word [ i : j ] ) i = j except : new_word . extend ( word [ i : ] ) break if word [ i ] == first and i < len ( word ) - 1 and word [ i + 1 ] == second : new_word . append ( first + second ) i += 2 else : new_word . append ( word [ i ] ) i += 1 new_word = tuple ( new_word ) word = new_word if len ( word ) == 1 : break else : pairs = get_pairs ( word ) # don't print end-of-word symbols if word [ - 1 ] == '</w>' : word = word [ : - 1 ] elif word [ - 1 ] . endswith ( '</w>' ) : word = word [ : - 1 ] + ( word [ - 1 ] . replace ( '</w>' , '' ) , ) if vocab : word = check_vocab_and_split ( word , bpe_codes_reverse , vocab , separator ) cache [ orig ] = word return word