(self, test)
Builds an LSTM graph in TensorFlow.
Builds an LSTM graph in TensorFlow.
Builds an LSTM graph in TensorFlow .
def build_graph(self, test):
        """
        Builds an LSTM graph in TensorFlow.
        """
        if test:
            self.batch_size = 1
            self.seq_len = 1

        ##
        # LSTM Cells
        ##

        lstm_cell = rnn_cell.BasicLSTMCell(self.cell_size)
        self.cell = rnn_cell.MultiRNNCell([lstm_cell] * self.num_layers)

        ##
        # Data
        ##

        # inputs and targets are 2D tensors of shape
        self.inputs = tf.placeholder(tf.int32, [self.batch_size, self.seq_len])
        self.targets = tf.placeholder(tf.int32, [self.batch_size, self.seq_len])
        self.initial_state = self.cell.zero_state(self.batch_size, tf.float32)

        ##
        # Variables
        ##
        with tf.variable_scope('lstm_vars'):
            self.ws = tf.get_variable('ws', [self.cell_size, self.vocab_size])
            self.bs = tf.get_variable('bs', [self.vocab_size])  # TODO: initializer?
            with tf.device('/cpu:0'): # put on CPU to parallelize for faster training/
                self.embeddings = tf.get_variable('embeddings', [self.vocab_size, self.cell_size])

                # get embeddings for all input words
                input_embeddings = tf.nn.embedding_lookup(self.embeddings, self.inputs)
                # The split splits this tensor into a seq_len long list of 3D tensors of shape
                # [batch_size, 1, rnn_size]. The squeeze removes the 1 dimension from the 1st axis
                # of each tensor
                inputs_split = tf.split(input_embeddings, self.seq_len, 1)
                inputs_split = [tf.squeeze(input_, [1]) for input_ in inputs_split]


                # inputs_split looks like this:
                # [
                #   tensor_<0>([
                #       [batchElt<0>_wordEmbedding<0>],
                #       ...,
                #       [batchElt<batch_size - 1>_wordEmbedding<0>]
                #   ]),
                #   ...,
                #   tensor_<seq_len - 1>([
                #       [batchElt<0>_wordEmbedding<seq_len - 1>],
                #       ...,
                #       [batchElt<batch_size - 1>_wordEmbedding<seq_len - 1>]
                #   ])
                # ]

        def loop(prev, _):
            prev = tf.matmul(prev, self.ws) + self.bs
            prev_symbol = tf.stop_gradient(tf.argmax(prev, 1))
            return tf.nn.embedding_lookup(self.embeddings, prev_symbol)

        lstm_outputs_split, self.final_state = seq2seq.rnn_decoder(inputs_split,
                                                                   self.initial_state,
                                                                   self.cell,
                                                                   loop_function=loop if test else None,
                                                                   scope='lstm_vars')
        lstm_outputs = tf.reshape(tf.concat(lstm_outputs_split, 1), [-1, self.cell_size])

        # outputs looks like this:
        # [
        #   tensor_<0>([
        #       [batchElt<0>_outputEmbedding<0>],
        #       ...,
        #       [batchElt<batch_size - 1>_outputEmbedding<0>]
        #   ]),
        #   ...,
        #   tensor_<seq_len - 1>([
        #       [batchElt<0>_outputEmbedding<seq_len - 1>],
        #       ...,
        #       [batchElt<batch_size - 1>_outputEmbedding<seq_len - 1>]
        #   ])
        # ]

        # output looks like this:
        # tensor([
        #     [batchElt<0>_outputEmbedding<0>],
        #     ...,
        #     [batchElt<0>_outputEmbedding<seq_len - 1>],
        #     [batchElt<1>_outputEmbedding<0>],
        #     ...,
        #     [batchElt<1>_outputEmbedding<seq_len - 1>],
        #     ...
        #     [batchElt<batch_size - 1>_outputEmbedding<0>],
        #     ...,
        #     [batchElt<batch_size - 1>_outputEmbedding<seq_len - 1>]
        # ])

        logits = tf.matmul(lstm_outputs, self.ws) + self.bs
        self.probs = tf.nn.softmax(logits)

        ##
        # Train
        ##

        total_loss = seq2seq.sequence_loss_by_example([logits],
                                                      [tf.reshape(self.targets, [-1])],
                                                      [tf.ones([self.batch_size * self.seq_len])],
                                                      self.vocab_size)
        self.loss = tf.reduce_sum(total_loss) / self.batch_size / self.seq_len

        self.global_step = tf.Variable(0, trainable=False, name='global_step')
        self.optimizer = tf.train.AdamOptimizer(learning_rate=c.L_RATE, name='optimizer')
        self.train_op = self.optimizer.minimize(self.loss,
                                                global_step=self.global_step,
                                                name='train_op')
def build_graph ( self , test ) : if test : self . batch_size = 1 self . seq_len = 1 ## # LSTM Cells ## lstm_cell = rnn_cell . BasicLSTMCell ( self . cell_size ) self . cell = rnn_cell . MultiRNNCell ( [ lstm_cell ] * self . num_layers ) ## # Data ## # inputs and targets are 2D tensors of shape self . inputs = tf . placeholder ( tf . int32 , [ self . batch_size , self . seq_len ] ) self . targets = tf . placeholder ( tf . int32 , [ self . batch_size , self . seq_len ] ) self . initial_state = self . cell . zero_state ( self . batch_size , tf . float32 ) ## # Variables ## with tf . variable_scope ( 'lstm_vars' ) : self . ws = tf . get_variable ( 'ws' , [ self . cell_size , self . vocab_size ] ) self . bs = tf . get_variable ( 'bs' , [ self . vocab_size ] ) # TODO: initializer? with tf . device ( '/cpu:0' ) : # put on CPU to parallelize for faster training/ self . embeddings = tf . get_variable ( 'embeddings' , [ self . vocab_size , self . cell_size ] ) # get embeddings for all input words input_embeddings = tf . nn . embedding_lookup ( self . embeddings , self . inputs ) # The split splits this tensor into a seq_len long list of 3D tensors of shape # [batch_size, 1, rnn_size]. The squeeze removes the 1 dimension from the 1st axis # of each tensor inputs_split = tf . split ( input_embeddings , self . seq_len , 1 ) inputs_split = [ tf . squeeze ( input_ , [ 1 ] ) for input_ in inputs_split ] # inputs_split looks like this: # [ #   tensor_<0>([ #       [batchElt<0>_wordEmbedding<0>], #       ..., #       [batchElt<batch_size - 1>_wordEmbedding<0>] #   ]), #   ..., #   tensor_<seq_len - 1>([ #       [batchElt<0>_wordEmbedding<seq_len - 1>], #       ..., #       [batchElt<batch_size - 1>_wordEmbedding<seq_len - 1>] #   ]) # ] def loop ( prev , _ ) : prev = tf . matmul ( prev , self . ws ) + self . bs prev_symbol = tf . stop_gradient ( tf . argmax ( prev , 1 ) ) return tf . nn . embedding_lookup ( self . embeddings , prev_symbol ) lstm_outputs_split , self . final_state = seq2seq . rnn_decoder ( inputs_split , self . initial_state , self . cell , loop_function = loop if test else None , scope = 'lstm_vars' ) lstm_outputs = tf . reshape ( tf . concat ( lstm_outputs_split , 1 ) , [ - 1 , self . cell_size ] ) # outputs looks like this: # [ #   tensor_<0>([ #       [batchElt<0>_outputEmbedding<0>], #       ..., #       [batchElt<batch_size - 1>_outputEmbedding<0>] #   ]), #   ..., #   tensor_<seq_len - 1>([ #       [batchElt<0>_outputEmbedding<seq_len - 1>], #       ..., #       [batchElt<batch_size - 1>_outputEmbedding<seq_len - 1>] #   ]) # ] # output looks like this: # tensor([ #     [batchElt<0>_outputEmbedding<0>], #     ..., #     [batchElt<0>_outputEmbedding<seq_len - 1>], #     [batchElt<1>_outputEmbedding<0>], #     ..., #     [batchElt<1>_outputEmbedding<seq_len - 1>], #     ... #     [batchElt<batch_size - 1>_outputEmbedding<0>], #     ..., #     [batchElt<batch_size - 1>_outputEmbedding<seq_len - 1>] # ]) logits = tf . matmul ( lstm_outputs , self . ws ) + self . bs self . probs = tf . nn . softmax ( logits ) ## # Train ## total_loss = seq2seq . sequence_loss_by_example ( [ logits ] , [ tf . reshape ( self . targets , [ - 1 ] ) ] , [ tf . ones ( [ self . batch_size * self . seq_len ] ) ] , self . vocab_size ) self . loss = tf . reduce_sum ( total_loss ) / self . batch_size / self . seq_len self . global_step = tf . Variable ( 0 , trainable = False , name = 'global_step' ) self . optimizer = tf . train . AdamOptimizer ( learning_rate = c . L_RATE , name = 'optimizer' ) self . train_op = self . optimizer . minimize ( self . loss , global_step = self . global_step , name = 'train_op' )