code: def _get_tokens(morph_fd):
      '''split a file-like into tokens: split on whitespace
  
      Note: this also strips newlines and comments
      '''
      for line in morph_fd:
          line = line.rstrip()   # remove \r\n
          line = line.split(';', 1)[0]  # strip comments
          squash_token = []  # quoted strings get squashed into one token
  
          if '<(' in line:  # skip spines, which exist on a single line
              assert ')>' in line, 'Missing end of spine'
              continue
  
          for token in line.replace('(', ' ( ').replace(')', ' ) ').split():
              if squash_token:
                  squash_token.append(token)
                  if token.endswith('"'):
                      token = ' '.join(squash_token)
                      squash_token = []
                      yield token
              elif token.startswith('"') and not token.endswith('"'):
                  squash_token.append(token)
              else:
                  yield tokef _get_tokens(morph_fd):
      '''split a file-like into tokens: split on whitespace
  
      Note: this also strips newlines and comments
      '''
      for line in morph_fd:
          line = line.rstrip()   # remove \r\n
          line = line.split(';', 1)[0]  # strip comments
          squash_token = []  # quoted strings get squashed into one token
  
          if '<(' in line:  # skip spines, which exist on a single line
              assert ')>' in line, 'Missing end of spine'
              continue
  
          for token in line.replace('(', ' ( ').replace(')', ' ) ').split():
              if squash_token:
                  squash_token.append(token)
                  if token.endswith('"'):
                      token = ' '.join(squash_token)
                      squash_token = []
                      yield token
              elif token.startswith('"') and not token.endswith('"'):
                  squash_token.append(token)
              else:
                  yield token
docs: split a file-like into tokens: split on whitespace
  
      Note: this also strips newlines and comments