(self, text)
Since ERB doesn't allow "<%" and other tags inside of ruby
        blocks we have to use a split approach here that fails for
        that too.
Since ERB doesn't allow "<%" and other tags inside of ruby
        blocks we have to use a split approach here that fails for
        that too.
Since ERB doesn t allow <% and other tags inside of ruby blocks we have to use a split approach here that fails for that too .
def get_tokens_unprocessed(self, text):
        """
        Since ERB doesn't allow "<%" and other tags inside of ruby
        blocks we have to use a split approach here that fails for
        that too.
        """
        tokens = self._block_re.split(text)
        tokens.reverse()
        state = idx = 0
        try:
            while True:
                # text
                if state == 0:
                    val = tokens.pop()
                    yield idx, Other, val
                    idx += len(val)
                    state = 1
                # block starts
                elif state == 1:
                    tag = tokens.pop()
                    # literals
                    if tag in ('<%%', '%%>'):
                        yield idx, Other, tag
                        idx += 3
                        state = 0
                    # comment
                    elif tag == '<%#':
                        yield idx, Comment.Preproc, tag
                        val = tokens.pop()
                        yield idx + 3, Comment, val
                        idx += 3 + len(val)
                        state = 2
                    # blocks or output
                    elif tag in ('<%', '<%=', '<%-'):
                        yield idx, Comment.Preproc, tag
                        idx += len(tag)
                        data = tokens.pop()
                        r_idx = 0
                        for r_idx, r_token, r_value in \
                            self.ruby_lexer.get_tokens_unprocessed(data):
                            yield r_idx + idx, r_token, r_value
                        idx += len(data)
                        state = 2
                    elif tag in ('%>', '-%>'):
                        yield idx, Error, tag
                        idx += len(tag)
                        state = 0
                    # % raw ruby statements
                    else:
                        yield idx, Comment.Preproc, tag[0]
                        r_idx = 0
                        for r_idx, r_token, r_value in \
                            self.ruby_lexer.get_tokens_unprocessed(tag[1:]):
                            yield idx + 1 + r_idx, r_token, r_value
                        idx += len(tag)
                        state = 0
                # block ends
                elif state == 2:
                    tag = tokens.pop()
                    if tag not in ('%>', '-%>'):
                        yield idx, Other, tag
                    else:
                        yield idx, Comment.Preproc, tag
                    idx += len(tag)
                    state = 0
        except IndexError:
            return
def get_tokens_unprocessed ( self , text ) : tokens = self . _block_re . split ( text ) tokens . reverse ( ) state = idx = 0 try : while True : # text if state == 0 : val = tokens . pop ( ) yield idx , Other , val idx += len ( val ) state = 1 # block starts elif state == 1 : tag = tokens . pop ( ) # literals if tag in ( '<%%' , '%%>' ) : yield idx , Other , tag idx += 3 state = 0 # comment elif tag == '<%#' : yield idx , Comment . Preproc , tag val = tokens . pop ( ) yield idx + 3 , Comment , val idx += 3 + len ( val ) state = 2 # blocks or output elif tag in ( '<%' , '<%=' , '<%-' ) : yield idx , Comment . Preproc , tag idx += len ( tag ) data = tokens . pop ( ) r_idx = 0 for r_idx , r_token , r_value in self . ruby_lexer . get_tokens_unprocessed ( data ) : yield r_idx + idx , r_token , r_value idx += len ( data ) state = 2 elif tag in ( '%>' , '-%>' ) : yield idx , Error , tag idx += len ( tag ) state = 0 # % raw ruby statements else : yield idx , Comment . Preproc , tag [ 0 ] r_idx = 0 for r_idx , r_token , r_value in self . ruby_lexer . get_tokens_unprocessed ( tag [ 1 : ] ) : yield idx + 1 + r_idx , r_token , r_value idx += len ( tag ) state = 0 # block ends elif state == 2 : tag = tokens . pop ( ) if tag not in ( '%>' , '-%>' ) : yield idx , Other , tag else : yield idx , Comment . Preproc , tag idx += len ( tag ) state = 0 except IndexError : return