(self, max_size=None, min_freq=1)
Initialize id2word & word2id based on self.freqdist
        max_size include 4 special tokens
Initialize id2word & word2id based on self.freqdist
        max_size include 4 special tokens
Initialize id2word & word2id based on self . freqdist max_size include 4 special tokens
def update(self, max_size=None, min_freq=1):
        """
        Initialize id2word & word2id based on self.freqdist
        max_size include 4 special tokens
        """

        # {0: '<pad>', 1: '<unk>', 2: '<sos>', 3: '<eos>'}
        self.id2word = {
            PAD_ID: PAD_TOKEN, UNK_ID: UNK_TOKEN,
            SOS_ID: SOS_TOKEN, EOS_ID: EOS_TOKEN
        }
        # {'<pad>': 0, '<unk>': 1, '<sos>': 2, '<eos>': 3}
        self.word2id = defaultdict(lambda: UNK_ID)  # Not in vocab => return UNK
        self.word2id.update({
            PAD_TOKEN: PAD_ID, UNK_TOKEN: UNK_ID,
            SOS_TOKEN: SOS_ID, EOS_TOKEN: EOS_ID
        })
        # self.word2id = {
        #     PAD_TOKEN: PAD_ID, UNK_TOKEN: UNK_ID,
        #     SOS_TOKEN: SOS_ID, EOS_TOKEN: EOS_ID
        # }

        vocab_size = 4
        min_freq = max(min_freq, 1)

        # Reset frequencies of special tokens
        # [...('<eos>', 0), ('<pad>', 0), ('<sos>', 0), ('<unk>', 0)]
        freqdist = self.freqdist.copy()
        special_freqdist = {token: freqdist[token]
                            for token in [PAD_TOKEN, UNK_TOKEN, SOS_TOKEN, EOS_TOKEN]}
        freqdist.subtract(special_freqdist)

        # Sort: by frequency, then alphabetically
        # Ex) freqdist = { 'a': 4,   'b': 5,   'c': 3 }
        #  =>   sorted = [('b', 5), ('a', 4), ('c', 3)]
        sorted_frequency_counter = sorted(freqdist.items(), key=lambda k_v: k_v[0])
        sorted_frequency_counter.sort(key=lambda k_v: k_v[1], reverse=True)

        for word, freq in sorted_frequency_counter:

            if freq < min_freq or vocab_size == max_size:
                break
            self.id2word[vocab_size] = word
            self.word2id[word] = vocab_size
            vocab_size += 1

        self.vocab_size = vocab_size
def update ( self , max_size = None , min_freq = 1 ) : # {0: '<pad>', 1: '<unk>', 2: '<sos>', 3: '<eos>'} self . id2word = { PAD_ID : PAD_TOKEN , UNK_ID : UNK_TOKEN , SOS_ID : SOS_TOKEN , EOS_ID : EOS_TOKEN } # {'<pad>': 0, '<unk>': 1, '<sos>': 2, '<eos>': 3} self . word2id = defaultdict ( lambda : UNK_ID ) # Not in vocab => return UNK self . word2id . update ( { PAD_TOKEN : PAD_ID , UNK_TOKEN : UNK_ID , SOS_TOKEN : SOS_ID , EOS_TOKEN : EOS_ID } ) # self.word2id = { #     PAD_TOKEN: PAD_ID, UNK_TOKEN: UNK_ID, #     SOS_TOKEN: SOS_ID, EOS_TOKEN: EOS_ID # } vocab_size = 4 min_freq = max ( min_freq , 1 ) # Reset frequencies of special tokens # [...('<eos>', 0), ('<pad>', 0), ('<sos>', 0), ('<unk>', 0)] freqdist = self . freqdist . copy ( ) special_freqdist = { token : freqdist [ token ] for token in [ PAD_TOKEN , UNK_TOKEN , SOS_TOKEN , EOS_TOKEN ] } freqdist . subtract ( special_freqdist ) # Sort: by frequency, then alphabetically # Ex) freqdist = { 'a': 4,   'b': 5,   'c': 3 } #  =>   sorted = [('b', 5), ('a', 4), ('c', 3)] sorted_frequency_counter = sorted ( freqdist . items ( ) , key = lambda k_v : k_v [ 0 ] ) sorted_frequency_counter . sort ( key = lambda k_v : k_v [ 1 ] , reverse = True ) for word , freq in sorted_frequency_counter : if freq < min_freq or vocab_size == max_size : break self . id2word [ vocab_size ] = word self . word2id [ word ] = vocab_size vocab_size += 1 self . vocab_size = vocab_size