(m)
Merge UTF16 surrogates, replace others
Merge UTF16 surrogates, replace others
Merge UTF16 surrogates replace others
def _fix_utf8(m):
    """Merge UTF16 surrogates, replace others"""
    u = m.group()
    if len(u) == 2:
        # merge into single symbol
        c1 = ord(u[0])
        c2 = ord(u[1])
        c = 0x10000 + ((c1 & 0x3FF) << 10) + (c2 & 0x3FF)
        return unichr(c)
    else:
        # use replacement symbol
        return REPLACEMENT_SYMBOL
def _fix_utf8 ( m ) : u = m . group ( ) if len ( u ) == 2 : # merge into single symbol c1 = ord ( u [ 0 ] ) c2 = ord ( u [ 1 ] ) c = 0x10000 + ( ( c1 & 0x3FF ) << 10 ) + ( c2 & 0x3FF ) return unichr ( c ) else : # use replacement symbol return REPLACEMENT_SYMBOL