(self, text, unfiltered=False)
return stream
Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.
Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.
Return an iterable of ( tokentype value ) pairs generated from text . If unfiltered is set to True the filtering mechanism is bypassed even if filters are defined .
def get_tokens(self, text, unfiltered=False):
        """
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.
        """
        if not isinstance(text, unicode):
            if self.encoding == 'guess':
                try:
                    text = text.decode('utf-8')
                    if text.startswith(u'\ufeff'):
                        text = text[len(u'\ufeff'):]
                except UnicodeDecodeError:
                    text = text.decode('latin1')
            elif self.encoding == 'chardet':
                try:
                    import chardet
                except ImportError:
                    raise ImportError('To enable chardet encoding guessing, '
                                      'please install the chardet library '
                                      'from http://chardet.feedparser.org/')
                # check for BOM first
                decoded = None
                for bom, encoding in _encoding_map:
                    if text.startswith(bom):
                        decoded = unicode(text[len(bom):], encoding,
                                          errors='replace')
                        break
                # no BOM found, so use chardet
                if decoded is None:
                    enc = chardet.detect(text[:1024]) # Guess using first 1KB
                    decoded = unicode(text, enc.get('encoding') or 'utf-8',
                                      errors='replace')
                text = decoded
            else:
                text = text.decode(self.encoding)
        # text now *is* a unicode string
        text = text.replace('\r\n', '\n')
        text = text.replace('\r', '\n')
        if self.stripall:
            text = text.strip()
        elif self.stripnl:
            text = text.strip('\n')
        if self.tabsize > 0:
            text = text.expandtabs(self.tabsize)
        if self.ensurenl and not text.endswith('\n'):
            text += '\n'

        def streamer():
            for i, t, v in self.get_tokens_unprocessed(text):
                yield t, v
        stream = streamer()
        if not unfiltered:
            stream = apply_filters(stream, self.filters, self)
        return stream
def get_tokens ( self , text , unfiltered = False ) : if not isinstance ( text , unicode ) : if self . encoding == 'guess' : try : text = text . decode ( 'utf-8' ) if text . startswith ( u'\ufeff' ) : text = text [ len ( u'\ufeff' ) : ] except UnicodeDecodeError : text = text . decode ( 'latin1' ) elif self . encoding == 'chardet' : try : import chardet except ImportError : raise ImportError ( 'To enable chardet encoding guessing, ' 'please install the chardet library ' 'from http://chardet.feedparser.org/' ) # check for BOM first decoded = None for bom , encoding in _encoding_map : if text . startswith ( bom ) : decoded = unicode ( text [ len ( bom ) : ] , encoding , errors = 'replace' ) break # no BOM found, so use chardet if decoded is None : enc = chardet . detect ( text [ : 1024 ] ) # Guess using first 1KB decoded = unicode ( text , enc . get ( 'encoding' ) or 'utf-8' , errors = 'replace' ) text = decoded else : text = text . decode ( self . encoding ) # text now *is* a unicode string text = text . replace ( '\r\n' , '\n' ) text = text . replace ( '\r' , '\n' ) if self . stripall : text = text . strip ( ) elif self . stripnl : text = text . strip ( '\n' ) if self . tabsize > 0 : text = text . expandtabs ( self . tabsize ) if self . ensurenl and not text . endswith ( '\n' ) : text += '\n' def streamer ( ) : for i , t , v in self . get_tokens_unprocessed ( text ) : yield t , v stream = streamer ( ) if not unfiltered : stream = apply_filters ( stream , self . filters , self ) return stream