(self, text, unfiltered=False)
return stream
Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.
Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.
Return an iterable of ( tokentype value ) pairs generated from text . If unfiltered is set to True the filtering mechanism is bypassed even if filters are defined .
def get_tokens(self, text, unfiltered=False):
        """
        Return an iterable of (tokentype, value) pairs generated from
        `text`. If `unfiltered` is set to `True`, the filtering mechanism
        is bypassed even if filters are defined.

        Also preprocess the text, i.e. expand tabs and strip it if
        wanted and applies registered filters.
        """
        if not isinstance(text, str):
            if self.encoding == 'guess':
                try:
                    text = text.decode('utf-8')
                    if text.startswith(u'\ufeff'):
                        text = text[len(u'\ufeff'):]
                except UnicodeDecodeError:
                    text = text.decode('latin1')
            else:
                text = text.decode(self.encoding)
        if self.stripall:
            text = text.strip()
        elif self.stripnl:
            text = text.strip('\n')
        if self.tabsize > 0:
            text = text.expandtabs(self.tabsize)
#        if not text.endswith('\n'):
#            text += '\n'

        def streamer():
            for i, t, v in self.get_tokens_unprocessed(text):
                yield t, v
        stream = streamer()
        if not unfiltered:
            stream = apply_filters(stream, self.filters, self)
        return stream
def get_tokens ( self , text , unfiltered = False ) : if not isinstance ( text , str ) : if self . encoding == 'guess' : try : text = text . decode ( 'utf-8' ) if text . startswith ( u'\ufeff' ) : text = text [ len ( u'\ufeff' ) : ] except UnicodeDecodeError : text = text . decode ( 'latin1' ) else : text = text . decode ( self . encoding ) if self . stripall : text = text . strip ( ) elif self . stripnl : text = text . strip ( '\n' ) if self . tabsize > 0 : text = text . expandtabs ( self . tabsize ) #        if not text.endswith('\n'): #            text += '\n' def streamer ( ) : for i , t , v in self . get_tokens_unprocessed ( text ) : yield t , v stream = streamer ( ) if not unfiltered : stream = apply_filters ( stream , self . filters , self ) return stream