(self)
Compute vocab, word index, and max length of stories and queries.
Compute vocab, word index, and max length of stories and queries.
Compute vocab word index and max length of stories and queries .
def compute_statistics(self):
        """
        Compute vocab, word index, and max length of stories and queries.
        """
        all_data = self.train_parsed + self.test_parsed
        vocab = sorted(reduce(lambda x, y: x | y, (set(s + q + [a]) for s, q, a in all_data)))
        self.vocab = vocab
        # Reserve 0 for masking via pad_sequences and self.vocab_size - 1 for <UNK> token
        self.vocab_size = len(vocab) + 2
        self.word_to_index = dict((c, i + 1) for i, c in enumerate(vocab))
        self.index_to_word = dict((i + 1, c) for i, c in enumerate(vocab))
        self.story_maxlen = max(list(map(len, (s for s, _, _ in all_data))))
        self.query_maxlen = max(list(map(len, (q for _, q, _ in all_data))))
def compute_statistics ( self ) : all_data = self . train_parsed + self . test_parsed vocab = sorted ( reduce ( lambda x , y : x | y , ( set ( s + q + [ a ] ) for s , q , a in all_data ) ) ) self . vocab = vocab # Reserve 0 for masking via pad_sequences and self.vocab_size - 1 for <UNK> token self . vocab_size = len ( vocab ) + 2 self . word_to_index = dict ( ( c , i + 1 ) for i , c in enumerate ( vocab ) ) self . index_to_word = dict ( ( i + 1 , c ) for i , c in enumerate ( vocab ) ) self . story_maxlen = max ( list ( map ( len , ( s for s , _ , _ in all_data ) ) ) ) self . query_maxlen = max ( list ( map ( len , ( q for _ , q , _ in all_data ) ) ) )