(smiles, tokens=None)
return tokens, token2idx, num_tokens
Returns list of unique tokens, token-2-index dictionary and
    number of unique tokens from the list of SMILES
    Args:
        smiles (list): list of SMILES strings to tokenize.
        tokens (string): string of tokens or None.
        If none will be extracted from dataset.
    Returns:
        tokens (list): list of unique tokens/SMILES alphabet.
        token2idx (dict): dictionary mapping token to its index.
        num_tokens (int): number of unique tokens.
Returns list of unique tokens, token-2-index dictionary and
    number of unique tokens from the list of SMILES
    Args:
        smiles (list): list of SMILES strings to tokenize.
        tokens (string): string of tokens or None.
        If none will be extracted from dataset.
    Returns:
        tokens (list): list of unique tokens/SMILES alphabet.
        token2idx (dict): dictionary mapping token to its index.
        num_tokens (int): number of unique tokens.
Returns list of unique tokens token - 2 - index dictionary and number of unique tokens from the list of SMILES Args : smiles ( list ) : list of SMILES strings to tokenize . tokens ( string ) : string of tokens or None . If none will be extracted from dataset . Returns : tokens ( list ) : list of unique tokens / SMILES alphabet . token2idx ( dict ) : dictionary mapping token to its index . num_tokens ( int ) : number of unique tokens .
def get_tokens(smiles, tokens=None):
    """
    Returns list of unique tokens, token-2-index dictionary and
    number of unique tokens from the list of SMILES
    Args:
        smiles (list): list of SMILES strings to tokenize.
        tokens (string): string of tokens or None.
        If none will be extracted from dataset.
    Returns:
        tokens (list): list of unique tokens/SMILES alphabet.
        token2idx (dict): dictionary mapping token to its index.
        num_tokens (int): number of unique tokens.
    """
    if tokens is None:
        tokens = list(set(''.join(smiles)))
        tokens = sorted(tokens)
        tokens = ''.join(tokens)
    token2idx = dict((token, i) for i, token in enumerate(tokens))
    num_tokens = len(tokens)
    return tokens, token2idx, num_tokens
def get_tokens ( smiles , tokens = None ) : if tokens is None : tokens = list ( set ( '' . join ( smiles ) ) ) tokens = sorted ( tokens ) tokens = '' . join ( tokens ) token2idx = dict ( ( token , i ) for i , token in enumerate ( tokens ) ) num_tokens = len ( tokens ) return tokens , token2idx , num_tokens