(cls, in_bytes, main_encoding="utf8",
                  embedded_encoding="windows-1252")
return b''.join(byte_chunks)
Fix characters from one encoding embedded in some other encoding.

        Currently the only situation supported is Windows-1252 (or its
        subset ISO-8859-1), embedded in UTF-8.

        The input must be a bytestring. If you've already converted
        the document to Unicode, you're too late.

        The output is a bytestring in which `embedded_encoding`
        characters have been converted to their `main_encoding`
        equivalents.
Fix characters from one encoding embedded in some other encoding.
Fix characters from one encoding embedded in some other encoding .
def detwingle(cls, in_bytes, main_encoding="utf8",
                  embedded_encoding="windows-1252"):
        """Fix characters from one encoding embedded in some other encoding.

        Currently the only situation supported is Windows-1252 (or its
        subset ISO-8859-1), embedded in UTF-8.

        The input must be a bytestring. If you've already converted
        the document to Unicode, you're too late.

        The output is a bytestring in which `embedded_encoding`
        characters have been converted to their `main_encoding`
        equivalents.
        """
        if embedded_encoding.replace('_', '-').lower() not in (
            'windows-1252', 'windows_1252'):
            raise NotImplementedError(
                "Windows-1252 and ISO-8859-1 are the only currently supported "
                "embedded encodings.")

        if main_encoding.lower() not in ('utf8', 'utf-8'):
            raise NotImplementedError(
                "UTF-8 is the only currently supported main encoding.")

        byte_chunks = []

        chunk_start = 0
        pos = 0
        while pos < len(in_bytes):
            byte = in_bytes[pos]
            if not isinstance(byte, int):
                # Python 2.x
                byte = ord(byte)
            if (byte >= cls.FIRST_MULTIBYTE_MARKER
                and byte <= cls.LAST_MULTIBYTE_MARKER):
                # This is the start of a UTF-8 multibyte character. Skip
                # to the end.
                for start, end, size in cls.MULTIBYTE_MARKERS_AND_SIZES:
                    if byte >= start and byte <= end:
                        pos += size
                        break
            elif byte >= 0x80 and byte in cls.WINDOWS_1252_TO_UTF8:
                # We found a Windows-1252 character!
                # Save the string up to this point as a chunk.
                byte_chunks.append(in_bytes[chunk_start:pos])

                # Now translate the Windows-1252 character into UTF-8
                # and add it as another, one-byte chunk.
                byte_chunks.append(cls.WINDOWS_1252_TO_UTF8[byte])
                pos += 1
                chunk_start = pos
            else:
                # Go on to the next character.
                pos += 1
        if chunk_start == 0:
            # The string is unchanged.
            return in_bytes
        else:
            # Store the final chunk.
            byte_chunks.append(in_bytes[chunk_start:])
        return b''.join(byte_chunks)
def detwingle ( cls , in_bytes , main_encoding = "utf8" , embedded_encoding = "windows-1252" ) : if embedded_encoding . replace ( '_' , '-' ) . lower ( ) not in ( 'windows-1252' , 'windows_1252' ) : raise NotImplementedError ( "Windows-1252 and ISO-8859-1 are the only currently supported " "embedded encodings." ) if main_encoding . lower ( ) not in ( 'utf8' , 'utf-8' ) : raise NotImplementedError ( "UTF-8 is the only currently supported main encoding." ) byte_chunks = [ ] chunk_start = 0 pos = 0 while pos < len ( in_bytes ) : byte = in_bytes [ pos ] if not isinstance ( byte , int ) : # Python 2.x byte = ord ( byte ) if ( byte >= cls . FIRST_MULTIBYTE_MARKER and byte <= cls . LAST_MULTIBYTE_MARKER ) : # This is the start of a UTF-8 multibyte character. Skip # to the end. for start , end , size in cls . MULTIBYTE_MARKERS_AND_SIZES : if byte >= start and byte <= end : pos += size break elif byte >= 0x80 and byte in cls . WINDOWS_1252_TO_UTF8 : # We found a Windows-1252 character! # Save the string up to this point as a chunk. byte_chunks . append ( in_bytes [ chunk_start : pos ] ) # Now translate the Windows-1252 character into UTF-8 # and add it as another, one-byte chunk. byte_chunks . append ( cls . WINDOWS_1252_TO_UTF8 [ byte ] ) pos += 1 chunk_start = pos else : # Go on to the next character. pos += 1 if chunk_start == 0 : # The string is unchanged. return in_bytes else : # Store the final chunk. byte_chunks . append ( in_bytes [ chunk_start : ] ) return b'' . join ( byte_chunks )