(sentence)
return [w for w in words if w]
Very basic tokenizer: split the sentence into a list of tokens.
Very basic tokenizer: split the sentence into a list of tokens.
Very basic tokenizer : split the sentence into a list of tokens .
def basic_tokenizer(sentence):
  """Very basic tokenizer: split the sentence into a list of tokens."""
  words = []
  for space_separated_fragment in sentence.strip().split():
    words.extend(_WORD_SPLIT.split(space_separated_fragment))
  return [w for w in words if w]
def basic_tokenizer ( sentence ) : words = [ ] for space_separated_fragment in sentence . strip ( ) . split ( ) : words . extend ( _WORD_SPLIT . split ( space_separated_fragment ) ) return [ w for w in words if w ]