docs: def do_tokenize(self, text):
          """
          >>> tk = Tokenizer(ngrams=1, lc=True, collapse_urls=True, collapse_mentions=True, collapse_hashtags=False, retain_punc_toks=True)
          >>> tk.do_tokenize('http://www.foo.com fast-forward hi there :) how?? U.S.A. @you whaaaaaaat? #awesome.')
          ['THIS_IS_A_URL', 'fast-forward', 'hi', 'there', ':)', 'how', '??', 'u.s.a', '.', 'THIS_IS_A_MENTION', 'what', '?', 'HASHTAG_awesome', '.']
          >>> tk = Tokenizer(rt_prefix=True)
          >>> tk.do_tokenize('RT hi there')
          ['RT_hi', 'RT_there']
          >>> tk.do_tokenize('hi there')
          ['hi', 'there']
          """
          text = text.lower() if self.lc else text
          if self.collapse_hashtags:
              text = re.sub('#\S+', 'THIS_IS_A_HASHTAG', text)
          else:
              text = re.sub('#(\S+)', r'HASHTAG_\1', text)
          if self.collapse_mentions:
              text = re.sub('@\S+', 'THIS_IS_A_MENTION', text)
          if self.collapse_urls:
              text = re.sub('http\S+', 'THIS_IS_A_URL', text)
          if self.limit_repeats:
              text = re.sub(r'(.)\1\1\1+', r'\1', text)
          if self.collapse_digits:
              text = re.sub(r'[0-9]+', '9', text)
          toks = []
          for tok in text.split():
              tok = re.sub(r'^(' + punc_re + '+)', r'\1 ', tok)
              tok = re.sub(r'(' + punc_re + '+)$', r' \1', tok)
              for subtok in tok.split():
                  if self.retain_punc_toks or re.search('\w', subtok):
                      toks.append(subtok)
          if self.rt_prefix:
              rt_text = 'rt' if self.lc else 'RT'
              if rt_text in toks:
                  toks.remove(rt_text)
                  toks = ['RT_' + t for t in toks]
          return toks