(self, codepage_dict=None, box_protect=True)
Load and initialise codepage tables.
Load and initialise codepage tables.
Load and initialise codepage tables .
def __init__(self, codepage_dict=None, box_protect=True):
        """Load and initialise codepage tables."""
        codepage_dict = codepage_dict or {
            int2byte(_i): _c
            for _i, _c in enumerate(DEFAULT_CODEPAGE)
        }
        # protect box drawing sequences under dbcs?
        self.box_protect = box_protect
        # lead and trail bytes
        self.lead = set()
        self.trail = set()
        # box-protection sets
        self._box_left = [set(), set()]
        self._box_right = [set(), set()]
        # main dictionary
        self._cp_to_unicode = {}
        # glyph subsitutes for printable ascii
        self._substitutes = {}
        dbcs_num_chars = 0
        for cp_point, unicode_cluster in iteritems(codepage_dict):
            # normalise clusters so we can match later
            unicode_cluster = unicodedata.normalize('NFC', unicode_cluster)
            # do not redefine printable ASCII, but substitute glyphs
            if (
                    cp_point in PRINTABLE_ASCII and
                    (len(unicode_cluster) > 1 or ord(unicode_cluster) != ord(cp_point))
                ):
                ascii_cp = unichr(ord(cp_point))
                self._cp_to_unicode[cp_point] = ascii_cp
                self._substitutes[cp_point] = unicode_cluster
            else:
                self._cp_to_unicode[cp_point] = unicode_cluster
            # track lead and trail bytes
            if len(cp_point) == 2:
                self.lead.add(cp_point[0:1])
                self.trail.add(cp_point[1:2])
                dbcs_num_chars += 1
            # track box drawing chars
            else:
                for i in (0, 1):
                    if unicode_cluster in _BOX_LEFT_UNICODE[i]:
                        self._box_left[i].add(cp_point[0:1])
                    if unicode_cluster in _BOX_RIGHT_UNICODE[i]:
                        self._box_right[i].add(cp_point[0:1])
        # fill up any undefined 1-byte codepoints
        for c in range(256):
            if int2byte(c) not in self._cp_to_unicode:
                self._cp_to_unicode[int2byte(c)] = u'\0'
        self._unicode_to_cp = dict((reversed(_item) for _item in iteritems(self._cp_to_unicode)))
        self._inverse_substitutes = dict((reversed(_item) for _item in iteritems(self._substitutes)))
        # keep set of clusters of more than one unicode code point
        self._unicode_clusters = set(
            _cluster for _cluster in self._unicode_to_cp if len(_cluster) > 0
        )
        # is the current codepage a double-byte codepage?
        self.dbcs = dbcs_num_chars > 0
def __init__ ( self , codepage_dict = None , box_protect = True ) : codepage_dict = codepage_dict or { int2byte ( _i ) : _c for _i , _c in enumerate ( DEFAULT_CODEPAGE ) } # protect box drawing sequences under dbcs? self . box_protect = box_protect # lead and trail bytes self . lead = set ( ) self . trail = set ( ) # box-protection sets self . _box_left = [ set ( ) , set ( ) ] self . _box_right = [ set ( ) , set ( ) ] # main dictionary self . _cp_to_unicode = { } # glyph subsitutes for printable ascii self . _substitutes = { } dbcs_num_chars = 0 for cp_point , unicode_cluster in iteritems ( codepage_dict ) : # normalise clusters so we can match later unicode_cluster = unicodedata . normalize ( 'NFC' , unicode_cluster ) # do not redefine printable ASCII, but substitute glyphs if ( cp_point in PRINTABLE_ASCII and ( len ( unicode_cluster ) > 1 or ord ( unicode_cluster ) != ord ( cp_point ) ) ) : ascii_cp = unichr ( ord ( cp_point ) ) self . _cp_to_unicode [ cp_point ] = ascii_cp self . _substitutes [ cp_point ] = unicode_cluster else : self . _cp_to_unicode [ cp_point ] = unicode_cluster # track lead and trail bytes if len ( cp_point ) == 2 : self . lead . add ( cp_point [ 0 : 1 ] ) self . trail . add ( cp_point [ 1 : 2 ] ) dbcs_num_chars += 1 # track box drawing chars else : for i in ( 0 , 1 ) : if unicode_cluster in _BOX_LEFT_UNICODE [ i ] : self . _box_left [ i ] . add ( cp_point [ 0 : 1 ] ) if unicode_cluster in _BOX_RIGHT_UNICODE [ i ] : self . _box_right [ i ] . add ( cp_point [ 0 : 1 ] ) # fill up any undefined 1-byte codepoints for c in range ( 256 ) : if int2byte ( c ) not in self . _cp_to_unicode : self . _cp_to_unicode [ int2byte ( c ) ] = u'\0' self . _unicode_to_cp = dict ( ( reversed ( _item ) for _item in iteritems ( self . _cp_to_unicode ) ) ) self . _inverse_substitutes = dict ( ( reversed ( _item ) for _item in iteritems ( self . _substitutes ) ) ) # keep set of clusters of more than one unicode code point self . _unicode_clusters = set ( _cluster for _cluster in self . _unicode_to_cp if len ( _cluster ) > 0 ) # is the current codepage a double-byte codepage? self . dbcs = dbcs_num_chars > 0