(self, ucs)
return clusters
Split unicode string into codepage-dependent clusters; preserve e-ascii clusters.
Split unicode string into codepage-dependent clusters; preserve e-ascii clusters.
Split unicode string into codepage - dependent clusters ; preserve e - ascii clusters .
def _split_unicode(self, ucs):
        """Split unicode string into codepage-dependent clusters; preserve e-ascii clusters."""
        # clusters are stored as C normal form (combine what can be combined)
        ucs = unicodedata.normalize('NFC', ucs)
        clusters = []
        while ucs:
            if ucs[0] == u'\0' and ord(ucs[1:2]) < 256:
                # preserve e-ascii clusters
                length = 2
            else:
                # if a cluster is matched, preserve that; otherwise, use single unicode point
                # i.e. if a perceived grapheme cluster is not contained in the codepage
                # it will be passed codepoint-by-codepoint (and likely ignored/raised/replaced)
                length = 1
                for cluster in self._unicode_clusters:
                    if ucs.startswith(cluster):
                        length = len(cluster)
            clusters.append(ucs[:length])
            ucs = ucs[length:]
        return clusters
def _split_unicode ( self , ucs ) : # clusters are stored as C normal form (combine what can be combined) ucs = unicodedata . normalize ( 'NFC' , ucs ) clusters = [ ] while ucs : if ucs [ 0 ] == u'\0' and ord ( ucs [ 1 : 2 ] ) < 256 : # preserve e-ascii clusters length = 2 else : # if a cluster is matched, preserve that; otherwise, use single unicode point # i.e. if a perceived grapheme cluster is not contained in the codepage # it will be passed codepoint-by-codepoint (and likely ignored/raised/replaced) length = 1 for cluster in self . _unicode_clusters : if ucs . startswith ( cluster ) : length = len ( cluster ) clusters . append ( ucs [ : length ] ) ucs = ucs [ length : ] return clusters