(
        self,
        list_of_tokens: List[str] = None,
        padding_token: str = "<pad>",
        unknown_token: str = "<unk>",
        bos_token: str = "<bos>",
        eos_token: str = "<eos>",
        reserved_tokens: List[str] = None,
        token_to_idx: Dict[str, int] = None,
    )
Instantiating Vocab class

        Args:
            list_of_tokens (List[str]): list of tokens is source of vocabulary. each token is not duplicate
            padding_token (str): the representation for padding token
            unknown_token (str): the representation for any unknown token
            bos_token (str): the representation for the special token of beginning-of-sequence token
            eos_token (str): the representation for the special token of end-of-sequence token
            reserved_tokens (List[str]): a list specifying additional tokens to be added to the vocabulary
            token_to_idx (Dict[str, int]): If not `None`, specifies the indices of tokens to be used by the vocabulary.
                                           Each token in `token_to_index` must be part of the Vocab and each index can
                                           only be associated with a single token. `token_to_idx` is not required to
                                           contain a mapping for all tokens. For example, it is valid to only set the
                                            `unknown_token` index to 10 (instead of the default of 0) with
                                           `token_to_idx = {'<unk>': 10}`, assuming that there are at least 10 tokens in
                                            the vocabulary.
Instantiating Vocab class
Instantiating Vocab class
def __init__(
        self,
        list_of_tokens: List[str] = None,
        padding_token: str = "<pad>",
        unknown_token: str = "<unk>",
        bos_token: str = "<bos>",
        eos_token: str = "<eos>",
        reserved_tokens: List[str] = None,
        token_to_idx: Dict[str, int] = None,
    ):
        """Instantiating Vocab class

        Args:
            list_of_tokens (List[str]): list of tokens is source of vocabulary. each token is not duplicate
            padding_token (str): the representation for padding token
            unknown_token (str): the representation for any unknown token
            bos_token (str): the representation for the special token of beginning-of-sequence token
            eos_token (str): the representation for the special token of end-of-sequence token
            reserved_tokens (List[str]): a list specifying additional tokens to be added to the vocabulary
            token_to_idx (Dict[str, int]): If not `None`, specifies the indices of tokens to be used by the vocabulary.
                                           Each token in `token_to_index` must be part of the Vocab and each index can
                                           only be associated with a single token. `token_to_idx` is not required to
                                           contain a mapping for all tokens. For example, it is valid to only set the
                                            `unknown_token` index to 10 (instead of the default of 0) with
                                           `token_to_idx = {'<unk>': 10}`, assuming that there are at least 10 tokens in
                                            the vocabulary.
        """
        self._unknown_token = unknown_token
        self._padding_token = padding_token
        self._bos_token = bos_token
        self._eos_token = eos_token
        self._reserved_tokens = reserved_tokens
        self._special_tokens = []

        for tkn in [
            self._unknown_token,
            self._padding_token,
            self._bos_token,
            self._eos_token,
        ]:
            if tkn:
                self._special_tokens.append(tkn)

        if self._reserved_tokens:
            self._special_tokens.extend(self._reserved_tokens)

        if list_of_tokens:
            self._special_tokens.extend(
                list(
                    filter(lambda elm: elm not in self._special_tokens, list_of_tokens)
                )
            )

        self._token_to_idx, self._idx_to_token = self._build(self._special_tokens)

        if token_to_idx:
            self._sort_index_according_to_user_specification(token_to_idx)

        self._embedding = None
def __init__ ( self , list_of_tokens : List [ str ] = None , padding_token : str = "<pad>" , unknown_token : str = "<unk>" , bos_token : str = "<bos>" , eos_token : str = "<eos>" , reserved_tokens : List [ str ] = None , token_to_idx : Dict [ str , int ] = None , ) : self . _unknown_token = unknown_token self . _padding_token = padding_token self . _bos_token = bos_token self . _eos_token = eos_token self . _reserved_tokens = reserved_tokens self . _special_tokens = [ ] for tkn in [ self . _unknown_token , self . _padding_token , self . _bos_token , self . _eos_token , ] : if tkn : self . _special_tokens . append ( tkn ) if self . _reserved_tokens : self . _special_tokens . extend ( self . _reserved_tokens ) if list_of_tokens : self . _special_tokens . extend ( list ( filter ( lambda elm : elm not in self . _special_tokens , list_of_tokens ) ) ) self . _token_to_idx , self . _idx_to_token = self . _build ( self . _special_tokens ) if token_to_idx : self . _sort_index_according_to_user_specification ( token_to_idx ) self . _embedding = None