docs: def generate_tokens(doc, regex=CRE_TOKEN, strip=True, nonwords=False):
      r"""Return a sequence of words or tokens, using a re.match iteratively through the str
  
      >>> doc = "John D. Rock\n\nObjective: \n\tSeeking a position as Software --Architect-- / _Project Lead_ that can utilize my expertise and"
      >>> doc += " experiences in business application development and proven records in delivering 90's software. \n\nSummary: \n\tSoftware Architect"
      >>> doc += " who has gone through several full product-delivery life cycles from requirements gathering to deployment / production, and"
      >>> doc += " skilled in all areas of software development from client-side JavaScript to database modeling. With strong experiences in:"
      >>> doc += " \n\tRequirements gathering and analysis."
      >>> len(list(generate_tokens(doc, strip=False, nonwords=True)))
      82
      >>> seq = list(generate_tokens(doc, strip=False, nonwords=False))
      >>> len(seq)
      70
      >>> '.' in seq or ':' in seq
      False
      >>> s = set(generate_tokens(doc, strip=False, nonwords=True))
      >>> all(t in s for t in ('D', '.', ':', '_Project', 'Lead_', "90's", "Architect", "product-delivery"))
      True
      """
      if isinstance(regex, basestring):
          regex = re.compile(regex)
      for w in regex.finditer(doc):
          if w:
              w = w.group()
              if strip:
                  w = w.strip(r'-_*`()}{' + r"'")
              if w and (nonwords or not re.match(r'^' + RE_NONWORD + '$', w)):
                  yield w