(self)
Load vocab and initialize buffers
        Input sentence batch is of dimension (vocab_size, max_sentence_length * batch_size)
        where each column is the 1-hot representation of a word and the first batch_size columns
        are the first words of each sentence.
Load vocab and initialize buffers
        Input sentence batch is of dimension (vocab_size, max_sentence_length * batch_size)
        where each column is the 1-hot representation of a word and the first batch_size columns
        are the first words of each sentence.
Load vocab and initialize buffers Input sentence batch is of dimension ( vocab_size max_sentence_length * batch_size ) where each column is the 1 - hot representation of a word and the first batch_size columns are the first words of each sentence .
def load_vocab(self):
        """
        Load vocab and initialize buffers
        Input sentence batch is of dimension (vocab_size, max_sentence_length * batch_size)
        where each column is the 1-hot representation of a word and the first batch_size columns
        are the first words of each sentence.
        """

        sentences = [sent['tokens'] for sent in self.iterSentences()]
        # Flatten list of list of words to one list of words
        words = [word for sentence in sentences for word in sentence]
        # Count words and keep words greater than threshold
        word_counts = Counter(words)

        vocab = [self.end_token] + \
                [word for word in list(word_counts.keys()) if word_counts[word] >= 5]
        self.vocab_size = len(vocab)
        self.vocab_to_index = dict((c, i) for i, c in enumerate(vocab))
        self.index_to_vocab = dict((i, c) for i, c in enumerate(vocab))

        # Compute optional bias vector for initializing final linear layer bias
        word_counts[self.end_token] = len(sentences)
        self.bias_init = np.array([1.0 * word_counts[self.index_to_vocab[i]]
                                   for i in self.index_to_vocab]).reshape((self.vocab_size, 1))
        self.bias_init /= np.sum(self.bias_init)
        self.bias_init = np.log(self.bias_init)
        self.bias_init -= np.max(self.bias_init)

        self.max_sentence_length = max(len(sent) for sent in sentences) + 1

        self.dev_image = self.be.iobuf(self.image_size)
        self.dev_imageT = self.be.empty(self.dev_image.shape[::-1])
        self.dev_X = self.be.iobuf((self.vocab_size, self.max_sentence_length))
        self.dev_y = self.be.iobuf((self.vocab_size, self.max_sentence_length + 1))
        # Create mask to deal with variable length sentences
        self.dev_y_mask = self.be.iobuf((self.vocab_size, self.max_sentence_length + 1))
        self.y_mask = np.zeros(self.dev_y_mask.shape,
                               dtype=np.uint8).reshape(self.vocab_size,
                                                       self.max_sentence_length + 1, -1)
        self.y_mask_reshape = self.y_mask.reshape(self.dev_y_mask.shape)

        self.dev_lbl = self.be.iobuf(self.max_sentence_length, dtype=np.int32)
        self.dev_lblT = self.be.empty(self.dev_lbl.shape[::-1])
        self.dev_lblflat = self.dev_lbl.reshape((1, self.dev_lbl.size))

        self.dev_y_lbl = self.be.iobuf(self.max_sentence_length + 1, dtype=np.int32)
        self.dev_y_lblT = self.be.empty(self.dev_y_lbl.shape[::-1])
        self.dev_y_lblflat = self.dev_y_lbl.reshape((1, self.dev_y_lbl.size))

        self.shape = [self.image_size, (self.vocab_size, self.max_sentence_length)]
        neon_logger.display("Vocab size: %d, Max sentence length: %d" % (self.vocab_size,
                                                                         self.max_sentence_length))
def load_vocab ( self ) : sentences = [ sent [ 'tokens' ] for sent in self . iterSentences ( ) ] # Flatten list of list of words to one list of words words = [ word for sentence in sentences for word in sentence ] # Count words and keep words greater than threshold word_counts = Counter ( words ) vocab = [ self . end_token ] + [ word for word in list ( word_counts . keys ( ) ) if word_counts [ word ] >= 5 ] self . vocab_size = len ( vocab ) self . vocab_to_index = dict ( ( c , i ) for i , c in enumerate ( vocab ) ) self . index_to_vocab = dict ( ( i , c ) for i , c in enumerate ( vocab ) ) # Compute optional bias vector for initializing final linear layer bias word_counts [ self . end_token ] = len ( sentences ) self . bias_init = np . array ( [ 1.0 * word_counts [ self . index_to_vocab [ i ] ] for i in self . index_to_vocab ] ) . reshape ( ( self . vocab_size , 1 ) ) self . bias_init /= np . sum ( self . bias_init ) self . bias_init = np . log ( self . bias_init ) self . bias_init -= np . max ( self . bias_init ) self . max_sentence_length = max ( len ( sent ) for sent in sentences ) + 1 self . dev_image = self . be . iobuf ( self . image_size ) self . dev_imageT = self . be . empty ( self . dev_image . shape [ : : - 1 ] ) self . dev_X = self . be . iobuf ( ( self . vocab_size , self . max_sentence_length ) ) self . dev_y = self . be . iobuf ( ( self . vocab_size , self . max_sentence_length + 1 ) ) # Create mask to deal with variable length sentences self . dev_y_mask = self . be . iobuf ( ( self . vocab_size , self . max_sentence_length + 1 ) ) self . y_mask = np . zeros ( self . dev_y_mask . shape , dtype = np . uint8 ) . reshape ( self . vocab_size , self . max_sentence_length + 1 , - 1 ) self . y_mask_reshape = self . y_mask . reshape ( self . dev_y_mask . shape ) self . dev_lbl = self . be . iobuf ( self . max_sentence_length , dtype = np . int32 ) self . dev_lblT = self . be . empty ( self . dev_lbl . shape [ : : - 1 ] ) self . dev_lblflat = self . dev_lbl . reshape ( ( 1 , self . dev_lbl . size ) ) self . dev_y_lbl = self . be . iobuf ( self . max_sentence_length + 1 , dtype = np . int32 ) self . dev_y_lblT = self . be . empty ( self . dev_y_lbl . shape [ : : - 1 ] ) self . dev_y_lblflat = self . dev_y_lbl . reshape ( ( 1 , self . dev_y_lbl . size ) ) self . shape = [ self . image_size , ( self . vocab_size , self . max_sentence_length ) ] neon_logger . display ( "Vocab size: %d, Max sentence length: %d" % ( self . vocab_size , self . max_sentence_length ) )