(self)
Builds the reinforcement learning tensorflow graph.
Builds the reinforcement learning tensorflow graph.
Builds the reinforcement learning tensorflow graph .
def build_graph(self):
    """Builds the reinforcement learning tensorflow graph."""

    tf.logging.info('Adding reward computation portion of the graph')
    with tf.name_scope('reward_computation'):
      self.reward_scores = tf.identity(self.reward_rnn(), name='reward_scores')

    tf.logging.info('Adding taking action portion of graph')
    with tf.name_scope('taking_action'):
      # Output of the q network gives the value of taking each action (playing
      # each note).
      self.action_scores = tf.identity(self.q_network(), name='action_scores')
      tf.summary.histogram(
          'action_scores', self.action_scores)

      # The action values for the G algorithm are computed differently.
      if self.algorithm == 'g':
        self.g_action_scores = self.action_scores + self.reward_scores

        # Compute predicted action, which is the argmax of the action scores.
        self.action_softmax = tf.nn.softmax(self.g_action_scores,
                                            name='action_softmax')
        self.predicted_actions = tf.one_hot(tf.argmax(self.g_action_scores,
                                                      dimension=1,
                                                      name='predicted_actions'),
                                            self.num_actions)
      else:
        # Compute predicted action, which is the argmax of the action scores.
        self.action_softmax = tf.nn.softmax(self.action_scores,
                                            name='action_softmax')
        self.predicted_actions = tf.one_hot(tf.argmax(self.action_scores,
                                                      dimension=1,
                                                      name='predicted_actions'),
                                            self.num_actions)

    tf.logging.info('Add estimating future rewards portion of graph')
    with tf.name_scope('estimating_future_rewards'):
      # The target q network is used to estimate the value of the best action at
      # the state resulting from the current action.
      self.next_action_scores = tf.stop_gradient(self.target_q_network())
      tf.summary.histogram(
          'target_action_scores', self.next_action_scores)

      # Rewards are observed from the environment and are fed in later.
      self.rewards = tf.placeholder(tf.float32, (None,), name='rewards')

      # Each algorithm is attempting to model future rewards with a different
      # function.
      if self.algorithm == 'psi':
        self.target_vals = tf.reduce_logsumexp(self.next_action_scores,
                                               reduction_indices=[1,])
      elif self.algorithm == 'g':
        self.g_normalizer = tf.reduce_logsumexp(self.reward_scores,
                                                reduction_indices=[1,])
        self.g_normalizer = tf.reshape(self.g_normalizer, [-1, 1])
        self.g_normalizer = tf.tile(self.g_normalizer, [1, self.num_actions])
        self.g_action_scores = tf.subtract(
            (self.next_action_scores + self.reward_scores), self.g_normalizer)
        self.target_vals = tf.reduce_logsumexp(self.g_action_scores,
                                               reduction_indices=[1,])
      else:
        # Use default based on Q learning.
        self.target_vals = tf.reduce_max(self.next_action_scores,
                                         reduction_indices=[1,])

      # Total rewards are the observed rewards plus discounted estimated future
      # rewards.
      self.future_rewards = self.rewards + self.discount_rate * self.target_vals

    tf.logging.info('Adding q value prediction portion of graph')
    with tf.name_scope('q_value_prediction'):
      # Action mask will be a one-hot encoding of the action the network
      # actually took.
      self.action_mask = tf.placeholder(tf.float32, (None, self.num_actions),
                                        name='action_mask')
      self.masked_action_scores = tf.reduce_sum(self.action_scores *
                                                self.action_mask,
                                                reduction_indices=[1,])

      temp_diff = self.masked_action_scores - self.future_rewards

      # Prediction error is the mean squared error between the reward the
      # network actually received for a given action, and what it expected to
      # receive.
      self.prediction_error = tf.reduce_mean(tf.square(temp_diff))

      # Compute gradients.
      self.params = tf.trainable_variables()
      self.gradients = self.optimizer.compute_gradients(self.prediction_error)

      # Clip gradients.
      for i, (grad, var) in enumerate(self.gradients):
        if grad is not None:
          self.gradients[i] = (tf.clip_by_norm(grad, 5), var)

      for grad, var in self.gradients:
        tf.summary.histogram(var.name, var)
        if grad is not None:
          tf.summary.histogram(var.name + '/gradients', grad)

      # Backprop.
      self.train_op = self.optimizer.apply_gradients(self.gradients)

    tf.logging.info('Adding target network update portion of graph')
    with tf.name_scope('target_network_update'):
      # Updates the target_q_network to be similar to the q_network based on
      # the target_network_update_rate.
      self.target_network_update = []
      for v_source, v_target in zip(self.q_network.variables(),
                                    self.target_q_network.variables()):
        # Equivalent to target = (1-alpha) * target + alpha * source
        update_op = v_target.assign_sub(self.target_network_update_rate *
                                        (v_target - v_source))
        self.target_network_update.append(update_op)
      self.target_network_update = tf.group(*self.target_network_update)

    tf.summary.scalar(
        'prediction_error', self.prediction_error)

    self.summarize = tf.summary.merge_all()
    self.no_op1 = tf.no_op()
def build_graph ( self ) : tf . logging . info ( 'Adding reward computation portion of the graph' ) with tf . name_scope ( 'reward_computation' ) : self . reward_scores = tf . identity ( self . reward_rnn ( ) , name = 'reward_scores' ) tf . logging . info ( 'Adding taking action portion of graph' ) with tf . name_scope ( 'taking_action' ) : # Output of the q network gives the value of taking each action (playing # each note). self . action_scores = tf . identity ( self . q_network ( ) , name = 'action_scores' ) tf . summary . histogram ( 'action_scores' , self . action_scores ) # The action values for the G algorithm are computed differently. if self . algorithm == 'g' : self . g_action_scores = self . action_scores + self . reward_scores # Compute predicted action, which is the argmax of the action scores. self . action_softmax = tf . nn . softmax ( self . g_action_scores , name = 'action_softmax' ) self . predicted_actions = tf . one_hot ( tf . argmax ( self . g_action_scores , dimension = 1 , name = 'predicted_actions' ) , self . num_actions ) else : # Compute predicted action, which is the argmax of the action scores. self . action_softmax = tf . nn . softmax ( self . action_scores , name = 'action_softmax' ) self . predicted_actions = tf . one_hot ( tf . argmax ( self . action_scores , dimension = 1 , name = 'predicted_actions' ) , self . num_actions ) tf . logging . info ( 'Add estimating future rewards portion of graph' ) with tf . name_scope ( 'estimating_future_rewards' ) : # The target q network is used to estimate the value of the best action at # the state resulting from the current action. self . next_action_scores = tf . stop_gradient ( self . target_q_network ( ) ) tf . summary . histogram ( 'target_action_scores' , self . next_action_scores ) # Rewards are observed from the environment and are fed in later. self . rewards = tf . placeholder ( tf . float32 , ( None , ) , name = 'rewards' ) # Each algorithm is attempting to model future rewards with a different # function. if self . algorithm == 'psi' : self . target_vals = tf . reduce_logsumexp ( self . next_action_scores , reduction_indices = [ 1 , ] ) elif self . algorithm == 'g' : self . g_normalizer = tf . reduce_logsumexp ( self . reward_scores , reduction_indices = [ 1 , ] ) self . g_normalizer = tf . reshape ( self . g_normalizer , [ - 1 , 1 ] ) self . g_normalizer = tf . tile ( self . g_normalizer , [ 1 , self . num_actions ] ) self . g_action_scores = tf . subtract ( ( self . next_action_scores + self . reward_scores ) , self . g_normalizer ) self . target_vals = tf . reduce_logsumexp ( self . g_action_scores , reduction_indices = [ 1 , ] ) else : # Use default based on Q learning. self . target_vals = tf . reduce_max ( self . next_action_scores , reduction_indices = [ 1 , ] ) # Total rewards are the observed rewards plus discounted estimated future # rewards. self . future_rewards = self . rewards + self . discount_rate * self . target_vals tf . logging . info ( 'Adding q value prediction portion of graph' ) with tf . name_scope ( 'q_value_prediction' ) : # Action mask will be a one-hot encoding of the action the network # actually took. self . action_mask = tf . placeholder ( tf . float32 , ( None , self . num_actions ) , name = 'action_mask' ) self . masked_action_scores = tf . reduce_sum ( self . action_scores * self . action_mask , reduction_indices = [ 1 , ] ) temp_diff = self . masked_action_scores - self . future_rewards # Prediction error is the mean squared error between the reward the # network actually received for a given action, and what it expected to # receive. self . prediction_error = tf . reduce_mean ( tf . square ( temp_diff ) ) # Compute gradients. self . params = tf . trainable_variables ( ) self . gradients = self . optimizer . compute_gradients ( self . prediction_error ) # Clip gradients. for i , ( grad , var ) in enumerate ( self . gradients ) : if grad is not None : self . gradients [ i ] = ( tf . clip_by_norm ( grad , 5 ) , var ) for grad , var in self . gradients : tf . summary . histogram ( var . name , var ) if grad is not None : tf . summary . histogram ( var . name + '/gradients' , grad ) # Backprop. self . train_op = self . optimizer . apply_gradients ( self . gradients ) tf . logging . info ( 'Adding target network update portion of graph' ) with tf . name_scope ( 'target_network_update' ) : # Updates the target_q_network to be similar to the q_network based on # the target_network_update_rate. self . target_network_update = [ ] for v_source , v_target in zip ( self . q_network . variables ( ) , self . target_q_network . variables ( ) ) : # Equivalent to target = (1-alpha) * target + alpha * source update_op = v_target . assign_sub ( self . target_network_update_rate * ( v_target - v_source ) ) self . target_network_update . append ( update_op ) self . target_network_update = tf . group ( * self . target_network_update ) tf . summary . scalar ( 'prediction_error' , self . prediction_error ) self . summarize = tf . summary . merge_all ( ) self . no_op1 = tf . no_op ( )