(self)
return tf.get_default_graph()
Our model takes in path segments with states and actions, and generates Q values.
        These Q values serve as predictions of the true reward.
        We can compare two segments and sum the Q values to get a prediction of a label
        of which segment is better. We then learn the weights for our model by comparing
        these labels with an authority (either a human or synthetic labeler).
Our model takes in path segments with states and actions, and generates Q values.
        These Q values serve as predictions of the true reward.
        We can compare two segments and sum the Q values to get a prediction of a label
        of which segment is better. We then learn the weights for our model by comparing
        these labels with an authority (either a human or synthetic labeler).
Our model takes in path segments with states and actions and generates Q values . These Q values serve as predictions of the true reward . We can compare two segments and sum the Q values to get a prediction of a label of which segment is better . We then learn the weights for our model by comparing these labels with an authority ( either a human or synthetic labeler ) .
def _build_model(self):
        """
        Our model takes in path segments with states and actions, and generates Q values.
        These Q values serve as predictions of the true reward.
        We can compare two segments and sum the Q values to get a prediction of a label
        of which segment is better. We then learn the weights for our model by comparing
        these labels with an authority (either a human or synthetic labeler).
        """
        # Set up observation placeholders
        self.segment_obs_placeholder = tf.placeholder(
            dtype=tf.float32, shape=(None, None) + self.obs_shape, name="obs_placeholder")
        self.segment_alt_obs_placeholder = tf.placeholder(
            dtype=tf.float32, shape=(None, None) + self.obs_shape, name="alt_obs_placeholder")

        self.segment_act_placeholder = tf.placeholder(
            dtype=tf.float32, shape=(None, None) + self.act_shape, name="act_placeholder")
        self.segment_alt_act_placeholder = tf.placeholder(
            dtype=tf.float32, shape=(None, None) + self.act_shape, name="alt_act_placeholder")


        # A vanilla multi-layer perceptron maps a (state, action) pair to a reward (Q-value)
        mlp = FullyConnectedMLP(self.obs_shape, self.act_shape)

        self.q_value = self._predict_rewards(self.segment_obs_placeholder, self.segment_act_placeholder, mlp)
        alt_q_value = self._predict_rewards(self.segment_alt_obs_placeholder, self.segment_alt_act_placeholder, mlp)

        # We use trajectory segments rather than individual (state, action) pairs because
        # video clips of segments are easier for humans to evaluate
        segment_reward_pred_left = tf.reduce_sum(self.q_value, axis=1)
        segment_reward_pred_right = tf.reduce_sum(alt_q_value, axis=1)
        reward_logits = tf.stack([segment_reward_pred_left, segment_reward_pred_right], axis=1)  # (batch_size, 2)

        self.labels = tf.placeholder(dtype=tf.int32, shape=(None,), name="comparison_labels")

        # delta = 1e-5
        # clipped_comparison_labels = tf.clip_by_value(self.comparison_labels, delta, 1.0-delta)

        data_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=reward_logits, labels=self.labels)

        self.loss_op = tf.reduce_mean(data_loss)

        global_step = tf.Variable(0, name='global_step', trainable=False)
        self.train_op = tf.train.AdamOptimizer().minimize(self.loss_op, global_step=global_step)

        return tf.get_default_graph()
def _build_model ( self ) : # Set up observation placeholders self . segment_obs_placeholder = tf . placeholder ( dtype = tf . float32 , shape = ( None , None ) + self . obs_shape , name = "obs_placeholder" ) self . segment_alt_obs_placeholder = tf . placeholder ( dtype = tf . float32 , shape = ( None , None ) + self . obs_shape , name = "alt_obs_placeholder" ) self . segment_act_placeholder = tf . placeholder ( dtype = tf . float32 , shape = ( None , None ) + self . act_shape , name = "act_placeholder" ) self . segment_alt_act_placeholder = tf . placeholder ( dtype = tf . float32 , shape = ( None , None ) + self . act_shape , name = "alt_act_placeholder" ) # A vanilla multi-layer perceptron maps a (state, action) pair to a reward (Q-value) mlp = FullyConnectedMLP ( self . obs_shape , self . act_shape ) self . q_value = self . _predict_rewards ( self . segment_obs_placeholder , self . segment_act_placeholder , mlp ) alt_q_value = self . _predict_rewards ( self . segment_alt_obs_placeholder , self . segment_alt_act_placeholder , mlp ) # We use trajectory segments rather than individual (state, action) pairs because # video clips of segments are easier for humans to evaluate segment_reward_pred_left = tf . reduce_sum ( self . q_value , axis = 1 ) segment_reward_pred_right = tf . reduce_sum ( alt_q_value , axis = 1 ) reward_logits = tf . stack ( [ segment_reward_pred_left , segment_reward_pred_right ] , axis = 1 ) # (batch_size, 2) self . labels = tf . placeholder ( dtype = tf . int32 , shape = ( None , ) , name = "comparison_labels" ) # delta = 1e-5 # clipped_comparison_labels = tf.clip_by_value(self.comparison_labels, delta, 1.0-delta) data_loss = tf . nn . sparse_softmax_cross_entropy_with_logits ( logits = reward_logits , labels = self . labels ) self . loss_op = tf . reduce_mean ( data_loss ) global_step = tf . Variable ( 0 , name = 'global_step' , trainable = False ) self . train_op = tf . train . AdamOptimizer ( ) . minimize ( self . loss_op , global_step = global_step ) return tf . get_default_graph ( )