docs: def _process_tokens(self, char):
          """ Process a token character.
              """
  
          if (char in self.WHITESPACE or char == self.COMMENT_START or
                  char in self.QUOTES or char in self.TOKENS):
  
              add_token = True
  
              # escaped chars, keep going
              if char == self.SPACE or char in self.TOKENS:
                  if self._escaped:
                      add_token = False
  
              # start of comment
              elif char == self.COMMENT_START:
                  self._state = self.ST_COMMENT
  
              # start of quoted string
              elif char in self.QUOTES:
                  if self._escaped:
                      # escaped, keep going
                      add_token = False
  
                  else:
                      self._state = self.ST_STRING
                      self._last_quote = char  # store for later quote matching
  
              if add_token:
                  # store token
                  self._new_token()
  
                  if char in self.TOKENS:
                      # store char as a new token
                      self._new_token([char])
  
                  return  # skip adding token char
  
          self._token_chars.append(char)