(segment, bpe_codes, vocab, separator, final=False)
Recursively split segment into smaller units (by reversing BPE merges)
    until all units are either in-vocabulary, or cannot be split futher.
Recursively split segment into smaller units (by reversing BPE merges)
    until all units are either in-vocabulary, or cannot be split futher.
Recursively split segment into smaller units ( by reversing BPE merges ) until all units are either in - vocabulary or cannot be split futher .
def recursive_split(segment, bpe_codes, vocab, separator, final=False):
    """Recursively split segment into smaller units (by reversing BPE merges)
    until all units are either in-vocabulary, or cannot be split futher."""

    try:
        if final:
            left, right = bpe_codes[segment + '</w>']
            right = right[:-4]
        else:
            left, right = bpe_codes[segment]
    except:
        #sys.stderr.write('cannot split {0} further.\n'.format(segment))
        yield segment
        return

    if left + separator in vocab:
        yield left
    else:
        for item in recursive_split(left, bpe_codes, vocab, separator, False):
            yield item

    if (final and right in vocab) or (not final and right + separator in vocab):
        yield right
    else:
        for item in recursive_split(right, bpe_codes, vocab, separator, final):
            yield item
def recursive_split ( segment , bpe_codes , vocab , separator , final = False ) : try : if final : left , right = bpe_codes [ segment + '</w>' ] right = right [ : - 4 ] else : left , right = bpe_codes [ segment ] except : #sys.stderr.write('cannot split {0} further.\n'.format(segment)) yield segment return if left + separator in vocab : yield left else : for item in recursive_split ( left , bpe_codes , vocab , separator , False ) : yield item if ( final and right in vocab ) or ( not final and right + separator in vocab ) : yield right else : for item in recursive_split ( right , bpe_codes , vocab , separator , final ) : yield item