(self, convai_dict, smoothing_freq=5)
return prefix2words
map BPE-prefix => dict(full_words beginning with BPE-prefix, associated words_counts)
map BPE-prefix => dict(full_words beginning with BPE-prefix, associated words_counts)
map BPE - prefix = > dict ( full_words beginning with BPE - prefix associated words_counts )
def get_prefix2words(self, convai_dict, smoothing_freq=5):
        """ map BPE-prefix => dict(full_words beginning with BPE-prefix, associated words_counts) """
        prefix2words = defaultdict(dict)
        for i in trange(len(convai_dict)):
            word = convai_dict[i]
            freq = convai_dict.freq[word] + smoothing_freq
            bpe_tokens = self.tokenizer.bpe(word).split(' ')
            prefix_id = self.tokenizer.convert_tokens_to_ids(bpe_tokens[0])
            prefix2words[prefix_id].update(dict([(word, freq)]))

        for prefix_id, words in prefix2words.items():
            total_counts = sum(words.values())
            prefix2words[prefix_id] = dict((word, count/total_counts) for word, count in words.items())

        return prefix2words
def get_prefix2words ( self , convai_dict , smoothing_freq = 5 ) : prefix2words = defaultdict ( dict ) for i in trange ( len ( convai_dict ) ) : word = convai_dict [ i ] freq = convai_dict . freq [ word ] + smoothing_freq bpe_tokens = self . tokenizer . bpe ( word ) . split ( ' ' ) prefix_id = self . tokenizer . convert_tokens_to_ids ( bpe_tokens [ 0 ] ) prefix2words [ prefix_id ] . update ( dict ( [ ( word , freq ) ] ) ) for prefix_id , words in prefix2words . items ( ) : total_counts = sum ( words . values ( ) ) prefix2words [ prefix_id ] = dict ( ( word , count / total_counts ) for word , count in words . items ( ) ) return prefix2words