code: def get_tokens_unprocessed(self, text):
          
          tokens = self._block_re.split(text)
          tokens.reverse()
          state = idx = 0
          try:
              while True:
                  # text
                  if state == 0:
                      val = tokens.pop()
                      yield idx, Other, val
                      idx += len(val)
                      state = 1
                  # block starts
                  elif state == 1:
                      tag = tokens.pop()
                      # literals
                      if tag in ('<%%', '%%>'):
                          yield idx, Other, tag
                          idx += 3
                          state = 0
                      # comment
                      elif tag == '<%#':
                          yield idx, Comment.Preproc, tag
                          val = tokens.pop()
                          yield idx + 3, Comment, val
                          idx += 3 + len(val)
                          state = 2
                      # blocks or output
                      elif tag in ('<%', '<%=', '<%-'):
                          yield idx, Comment.Preproc, tag
                          idx += len(tag)
                          data = tokens.pop()
                          r_idx = 0
                          for r_idx, r_token, r_value in \
                                  self.ruby_lexer.get_tokens_unprocessed(data):
                              yield r_idx + idx, r_token, r_value
                          idx += len(data)
                          state = 2
                      elif tag in ('%>', '-%>'):
                          yield idx, Error, tag
                          idx += len(tag)
                          state = 0
                      # % raw ruby statements
                      else:
                          yield idx, Comment.Preproc, tag[0]
                          r_idx = 0
                          for r_idx, r_token, r_value in \
                                  self.ruby_lexer.get_tokens_unprocessed(tag[1:]):
                              yield idx + 1 + r_idx, r_token, r_value
                          idx += len(tag)
                          state = 0
                  # block ends
                  elif state == 2:
                      tag = tokens.pop()
                      if tag not in ('%>', '-%>'):
                          yield idx, Other, tag
                      else:
                          yield idx, Comment.Preproc, tag
                      idx += len(tag)
                      state = 0
          except IndexError:
              return
docs: Since ERB doesn't allow "<%" and other tags inside of ruby
          blocks we have to use a split approach here that fails for
          that too.