code: def tokenize(self, string):
          
          it = colorise.compat.ifilter(None, self._pattern.finditer(string))
  
          try:
              t = colorise.compat.next(it)
          except StopIteration:
              yield string, False
              return
  
          pos, buf, lm, escapeflag = -1, '', -1, False
  
          # Check if we need to yield any starting text
          if t.start() > 0:
              yield string[:t.start()], False
              pos = t.start()
  
          it = itertools.chain([t], it)
  
          for m in it:
              start = m.start()
              e, s = m.group(2) or '', m.group(3)
              escaped = e.count(self._ESCAPE) % 2 != 0
  
              if escaped:
                  buf += string[pos:m.end(2)-1] + s
                  escapeflag = True
              else:
                  buf += string[pos:m.start(3)]
  
                  if buf:
                      yield buf, escapeflag
                      buf = ''
                      escapeflag = False
  
                  if lm == start:
                      yield '', False
  
                  yield s, False
                  lm = m.end()
  
              pos = m.end()
  
          if buf:
              yield buf, escapeflag
              escapeflag = False
  
          if pos < len(string):
              yield string[pos:], False
docs: Tokenize a string and return an iterator over its tokens.