(string, tokenizer=None)
Map string to a list of tokens.

        Arguments:
            string(str): String to be tokenized.
            token(object): Tokenizer object.
            tokenizer (function) : Tokenizer function.

        Returns:
            list : A list of tokens
Map string to a list of tokens.
Map string to a list of tokens .
def get_tokens(string, tokenizer=None):
        """
        Map string to a list of tokens.

        Arguments:
            string(str): String to be tokenized.
            token(object): Tokenizer object.
            tokenizer (function) : Tokenizer function.

        Returns:
            list : A list of tokens
        """
        # (if tokenizer is None, we have a list of characters)
        if tokenizer is None:
            return string
        else:
            return tokenizer(string)
def get_tokens ( string , tokenizer = None ) : # (if tokenizer is None, we have a list of characters) if tokenizer is None : return string else : return tokenizer ( string )