(sentence)
return [token for span in space_tokenizer(sentence) for
            token in symbol_tokenizer.split(span) if token]
The symbol tokenizer extends the :func:`space_tokenizer` by separating alphanumerics.

    Separates alphanumeric Unicode character sequences in already space-split tokens.
The symbol tokenizer extends the :func:`space_tokenizer` by separating alphanumerics.
The symbol tokenizer extends the : func : space_tokenizer by separating alphanumerics .
def symbol_tokenizer(sentence):
    """
    The symbol tokenizer extends the :func:`space_tokenizer` by separating alphanumerics.

    Separates alphanumeric Unicode character sequences in already space-split tokens.
    """
    return [token for span in space_tokenizer(sentence) for
            token in symbol_tokenizer.split(span) if token]
def symbol_tokenizer ( sentence ) : return [ token for span in space_tokenizer ( sentence ) for token in symbol_tokenizer . split ( span ) if token ]