(self)
return word_perplexity
Perform importance sampling to get tighter bound
Perform importance sampling to get tighter bound
Perform importance sampling to get tighter bound
def importance_sample(self):
        ''' Perform importance sampling to get tighter bound
        '''
        self.model.eval()
        weight_history = []
        n_total_words = 0
        kl_div_history = []
        for batch_i, (conversations, conversation_length, sentence_length) \
                in enumerate(tqdm(self.eval_data_loader, ncols=80)):
            # conversations: (batch_size) list of conversations
            #   conversation: list of sentences
            #   sentence: list of tokens
            # conversation_length: list of int
            # sentence_length: (batch_size) list of conversation list of sentence_lengths

            target_conversations = [conv[1:] for conv in conversations]

            # flatten input and target conversations
            sentences = [sent for conv in conversations for sent in conv]
            input_conversation_length = [l - 1 for l in conversation_length]
            target_sentences = [sent for conv in target_conversations for sent in conv]
            target_sentence_length = [l for len_list in sentence_length for l in len_list[1:]]
            sentence_length = [l for len_list in sentence_length for l in len_list]

            # n_words += sum([len([word for word in sent if word != PAD_ID]) for sent in target_sentences])
            with torch.no_grad():
                sentences = to_var(torch.LongTensor(sentences))
                sentence_length = to_var(torch.LongTensor(sentence_length))
                input_conversation_length = to_var(
                    torch.LongTensor(input_conversation_length))
                target_sentences = to_var(torch.LongTensor(target_sentences))
                target_sentence_length = to_var(torch.LongTensor(target_sentence_length))

            # treat whole batch as one data sample
            weights = []
            for j in range(self.config.importance_sample):
                sentence_logits, kl_div, log_p_z, log_q_zx = self.model(
                    sentences,
                    sentence_length,
                    input_conversation_length,
                    target_sentences)

                recon_loss, n_words = masked_cross_entropy(
                    sentence_logits,
                    target_sentences,
                    target_sentence_length)

                log_w = (-recon_loss.sum() + log_p_z - log_q_zx).data
                weights.append(log_w)
                if j == 0:
                    n_total_words += n_words.item()
                    kl_div_history.append(kl_div.item())

            # weights: [n_samples]
            weights = torch.stack(weights, 0)
            m = np.floor(weights.max())
            weights = np.log(torch.exp(weights - m).sum())
            weights = m + weights - np.log(self.config.importance_sample)
            weight_history.append(weights)

        print(f'Number of words: {n_total_words}')
        bits_per_word = -np.sum(weight_history) / n_total_words
        print(f'Bits per word: {bits_per_word:.3f}')
        word_perplexity = np.exp(bits_per_word)

        epoch_kl_div = np.sum(kl_div_history) / n_total_words

        print_str = f'Word perplexity upperbound using {self.config.importance_sample} importance samples: {word_perplexity:.3f}, kl_div: {epoch_kl_div:.3f}\n'
        print(print_str)

        return word_perplexity
def importance_sample ( self ) : self . model . eval ( ) weight_history = [ ] n_total_words = 0 kl_div_history = [ ] for batch_i , ( conversations , conversation_length , sentence_length ) in enumerate ( tqdm ( self . eval_data_loader , ncols = 80 ) ) : # conversations: (batch_size) list of conversations #   conversation: list of sentences #   sentence: list of tokens # conversation_length: list of int # sentence_length: (batch_size) list of conversation list of sentence_lengths target_conversations = [ conv [ 1 : ] for conv in conversations ] # flatten input and target conversations sentences = [ sent for conv in conversations for sent in conv ] input_conversation_length = [ l - 1 for l in conversation_length ] target_sentences = [ sent for conv in target_conversations for sent in conv ] target_sentence_length = [ l for len_list in sentence_length for l in len_list [ 1 : ] ] sentence_length = [ l for len_list in sentence_length for l in len_list ] # n_words += sum([len([word for word in sent if word != PAD_ID]) for sent in target_sentences]) with torch . no_grad ( ) : sentences = to_var ( torch . LongTensor ( sentences ) ) sentence_length = to_var ( torch . LongTensor ( sentence_length ) ) input_conversation_length = to_var ( torch . LongTensor ( input_conversation_length ) ) target_sentences = to_var ( torch . LongTensor ( target_sentences ) ) target_sentence_length = to_var ( torch . LongTensor ( target_sentence_length ) ) # treat whole batch as one data sample weights = [ ] for j in range ( self . config . importance_sample ) : sentence_logits , kl_div , log_p_z , log_q_zx = self . model ( sentences , sentence_length , input_conversation_length , target_sentences ) recon_loss , n_words = masked_cross_entropy ( sentence_logits , target_sentences , target_sentence_length ) log_w = ( - recon_loss . sum ( ) + log_p_z - log_q_zx ) . data weights . append ( log_w ) if j == 0 : n_total_words += n_words . item ( ) kl_div_history . append ( kl_div . item ( ) ) # weights: [n_samples] weights = torch . stack ( weights , 0 ) m = np . floor ( weights . max ( ) ) weights = np . log ( torch . exp ( weights - m ) . sum ( ) ) weights = m + weights - np . log ( self . config . importance_sample ) weight_history . append ( weights ) print ( f'Number of words: {n_total_words}' ) bits_per_word = - np . sum ( weight_history ) / n_total_words print ( f'Bits per word: {bits_per_word:.3f}' ) word_perplexity = np . exp ( bits_per_word ) epoch_kl_div = np . sum ( kl_div_history ) / n_total_words print_str = f'Word perplexity upperbound using {self.config.importance_sample} importance samples: {word_perplexity:.3f}, kl_div: {epoch_kl_div:.3f}\n' print ( print_str ) return word_perplexity