(tokens, vocab=None)
Construct vocabulary from the given tokens.

        Arguments:
            tokens(list): List of tokens.
            vocab:  (Default value = None)

        Returns:
            python.set : A set of unique tokens
Construct vocabulary from the given tokens.
Construct vocabulary from the given tokens .
def get_vocab(tokens, vocab=None):
        """
        Construct vocabulary from the given tokens.

        Arguments:
            tokens(list): List of tokens.
            vocab:  (Default value = None)

        Returns:
            python.set : A set of unique tokens
        """
        # (if vocab is not None, we check that it contains all tokens)
        if vocab is None:
            return set(tokens)
        else:
            vocab = set(vocab)
            assert vocab >= set(tokens), "the predefined vocab must contain all the tokens"
            return vocab
def get_vocab ( tokens , vocab = None ) : # (if vocab is not None, we check that it contains all tokens) if vocab is None : return set ( tokens ) else : vocab = set ( vocab ) assert vocab >= set ( tokens ) , "the predefined vocab must contain all the tokens" return vocab