code: def _split_on_punctuation(self, tokens: List[str]) -> List[str]:
          
          punctuation = []
          punctuation.extend(self.terminators)
          if self.allow_redirection:
              punctuation.extend(constants.REDIRECTION_CHARS)
  
          punctuated_tokens = []
  
          for cur_initial_token in tokens:
  
              # Save tokens up to 1 character in length or quoted tokens. No need to parse these.
              if len(cur_initial_token) <= 1 or cur_initial_token[0] in constants.QUOTES:
                  punctuated_tokens.append(cur_initial_token)
                  continue
  
              # Iterate over each character in this token
              cur_index = 0
              cur_char = cur_initial_token[cur_index]
  
              # Keep track of the token we are building
              new_token = ''
  
              while True:
                  if cur_char not in punctuation:
  
                      # Keep appending to new_token until we hit a punctuation char
                      while cur_char not in punctuation:
                          new_token += cur_char
                          cur_index += 1
                          if cur_index < len(cur_initial_token):
                              cur_char = cur_initial_token[cur_index]
                          else:
                              break
  
                  else:
                      cur_punc = cur_char
  
                      # Keep appending to new_token until we hit something other than cur_punc
                      while cur_char == cur_punc:
                          new_token += cur_char
                          cur_index += 1
                          if cur_index < len(cur_initial_token):
                              cur_char = cur_initial_token[cur_index]
                          else:
                              break
  
                  # Save the new token
                  punctuated_tokens.append(new_token)
                  new_token = ''
  
                  # Check if we've viewed all characters
                  if cur_index >= len(cur_initial_token):
                      break
  
          return punctuated_tokens
docs: Further splits tokens from a command line using punctuation characters
  
          Punctuation characters are treated as word breaks when they are in
          unquoted strings. Each run of punctuation characters is treated as a
          single token.
  
          :param tokens: the tokens as parsed by shlex
          :return: the punctuated tokens