(self, m)
Calculate new step sizes.
        Args:
            m: Condition
Calculate new step sizes.
        Args:
            m: Condition
Calculate new step sizes . Args : m : Condition
def _stepadjust(self, m):
        """
        Calculate new step sizes.
        Args:
            m: Condition
        """

        # Compute values under Laplace approximation. This is the policy
        # that the previous samples were actually drawn from under the
        # dynamics that were estimated from the previous samples.
        prev_laplace_obj, prev_laplace_kl = self._estimate_cost(
            self.prev[m].traj_distr, self.prev[m].traj_info, self.prev[m].pol_info, m
        )
        # This is the policy that we just used under the dynamics that
        # were estimated from the previous samples (so this is the cost
        # we thought we would have).
        new_pred_laplace_obj, new_pred_laplace_kl = self._estimate_cost(
            self.cur[m].traj_distr, self.prev[m].traj_info, self.prev[m].pol_info, m
        )

        # This is the actual cost we have under the current trajectory
        # based on the latest samples.
        new_actual_laplace_obj, new_actual_laplace_kl = self._estimate_cost(
            self.cur[m].traj_distr, self.cur[m].traj_info, self.cur[m].pol_info, m
        )

        # Measure the entropy of the current trajectory (for printout).
        ent = self._measure_ent(m)

        # Compute actual objective values based on the samples.
        prev_mc_obj = np.mean(np.sum(self.prev[m].cs, axis=1), axis=0)
        new_mc_obj = np.mean(np.sum(self.cur[m].cs, axis=1), axis=0)

        # Compute sample-based estimate of KL divergence between policy
        # and trajectories.
        new_mc_kl = self._policy_kl(m)[0]
        if self.iteration_count >= 1 and self.prev[m].sample_list:
            prev_mc_kl = self._policy_kl(m, prev=True)[0]
        else:
            prev_mc_kl = np.zeros_like(new_mc_kl)

        # Compute full policy KL divergence objective terms by applying
        # the Lagrange multipliers.
        pol_wt = self.cur[m].pol_info.pol_wt
        prev_laplace_kl_sum = np.sum(prev_laplace_kl * pol_wt)
        new_pred_laplace_kl_sum = np.sum(new_pred_laplace_kl * pol_wt)
        new_actual_laplace_kl_sum = np.sum(new_actual_laplace_kl * pol_wt)
        prev_mc_kl_sum = np.sum(prev_mc_kl * pol_wt)
        new_mc_kl_sum = np.sum(new_mc_kl * pol_wt)

        LOGGER.debug(
            'Trajectory step: ent: %f cost: %f -> %f KL: %f -> %f',
            ent, prev_mc_obj, new_mc_obj, prev_mc_kl_sum, new_mc_kl_sum
        )

        # Compute predicted and actual improvement.
        predicted_impr = np.sum(prev_laplace_obj) + prev_laplace_kl_sum - \
                np.sum(new_pred_laplace_obj) - new_pred_laplace_kl_sum
        actual_impr = np.sum(prev_laplace_obj) + prev_laplace_kl_sum - \
                np.sum(new_actual_laplace_obj) - new_actual_laplace_kl_sum

        # Print improvement details.
        LOGGER.debug('Previous cost: Laplace: %f MC: %f',
                     np.sum(prev_laplace_obj), prev_mc_obj)
        LOGGER.debug('Predicted new cost: Laplace: %f MC: %f',
                     np.sum(new_pred_laplace_obj), new_mc_obj)
        LOGGER.debug('Actual new cost: Laplace: %f MC: %f',
                     np.sum(new_actual_laplace_obj), new_mc_obj)
        LOGGER.debug('Previous KL: Laplace: %f MC: %f',
                     np.sum(prev_laplace_kl), np.sum(prev_mc_kl))
        LOGGER.debug('Predicted new KL: Laplace: %f MC: %f',
                     np.sum(new_pred_laplace_kl), np.sum(new_mc_kl))
        LOGGER.debug('Actual new KL: Laplace: %f MC: %f',
                     np.sum(new_actual_laplace_kl), np.sum(new_mc_kl))
        LOGGER.debug('Previous w KL: Laplace: %f MC: %f',
                     prev_laplace_kl_sum, prev_mc_kl_sum)
        LOGGER.debug('Predicted w new KL: Laplace: %f MC: %f',
                     new_pred_laplace_kl_sum, new_mc_kl_sum)
        LOGGER.debug('Actual w new KL: Laplace %f MC: %f',
                     new_actual_laplace_kl_sum, new_mc_kl_sum)
        LOGGER.debug('Predicted/actual improvement: %f / %f',
                     predicted_impr, actual_impr)

        # Compute actual KL step taken at last iteration.
        actual_step = self.cur[m].traj_info.last_kl_step / \
                (self._hyperparams['kl_step'] * self.T)
        if actual_step < self.cur[m].step_mult:
            self.cur[m].step_mult = max(actual_step,
                                        self._hyperparams['min_step_mult'])

        self._set_new_mult(predicted_impr, actual_impr, m)
def _stepadjust ( self , m ) : # Compute values under Laplace approximation. This is the policy # that the previous samples were actually drawn from under the # dynamics that were estimated from the previous samples. prev_laplace_obj , prev_laplace_kl = self . _estimate_cost ( self . prev [ m ] . traj_distr , self . prev [ m ] . traj_info , self . prev [ m ] . pol_info , m ) # This is the policy that we just used under the dynamics that # were estimated from the previous samples (so this is the cost # we thought we would have). new_pred_laplace_obj , new_pred_laplace_kl = self . _estimate_cost ( self . cur [ m ] . traj_distr , self . prev [ m ] . traj_info , self . prev [ m ] . pol_info , m ) # This is the actual cost we have under the current trajectory # based on the latest samples. new_actual_laplace_obj , new_actual_laplace_kl = self . _estimate_cost ( self . cur [ m ] . traj_distr , self . cur [ m ] . traj_info , self . cur [ m ] . pol_info , m ) # Measure the entropy of the current trajectory (for printout). ent = self . _measure_ent ( m ) # Compute actual objective values based on the samples. prev_mc_obj = np . mean ( np . sum ( self . prev [ m ] . cs , axis = 1 ) , axis = 0 ) new_mc_obj = np . mean ( np . sum ( self . cur [ m ] . cs , axis = 1 ) , axis = 0 ) # Compute sample-based estimate of KL divergence between policy # and trajectories. new_mc_kl = self . _policy_kl ( m ) [ 0 ] if self . iteration_count >= 1 and self . prev [ m ] . sample_list : prev_mc_kl = self . _policy_kl ( m , prev = True ) [ 0 ] else : prev_mc_kl = np . zeros_like ( new_mc_kl ) # Compute full policy KL divergence objective terms by applying # the Lagrange multipliers. pol_wt = self . cur [ m ] . pol_info . pol_wt prev_laplace_kl_sum = np . sum ( prev_laplace_kl * pol_wt ) new_pred_laplace_kl_sum = np . sum ( new_pred_laplace_kl * pol_wt ) new_actual_laplace_kl_sum = np . sum ( new_actual_laplace_kl * pol_wt ) prev_mc_kl_sum = np . sum ( prev_mc_kl * pol_wt ) new_mc_kl_sum = np . sum ( new_mc_kl * pol_wt ) LOGGER . debug ( 'Trajectory step: ent: %f cost: %f -> %f KL: %f -> %f' , ent , prev_mc_obj , new_mc_obj , prev_mc_kl_sum , new_mc_kl_sum ) # Compute predicted and actual improvement. predicted_impr = np . sum ( prev_laplace_obj ) + prev_laplace_kl_sum - np . sum ( new_pred_laplace_obj ) - new_pred_laplace_kl_sum actual_impr = np . sum ( prev_laplace_obj ) + prev_laplace_kl_sum - np . sum ( new_actual_laplace_obj ) - new_actual_laplace_kl_sum # Print improvement details. LOGGER . debug ( 'Previous cost: Laplace: %f MC: %f' , np . sum ( prev_laplace_obj ) , prev_mc_obj ) LOGGER . debug ( 'Predicted new cost: Laplace: %f MC: %f' , np . sum ( new_pred_laplace_obj ) , new_mc_obj ) LOGGER . debug ( 'Actual new cost: Laplace: %f MC: %f' , np . sum ( new_actual_laplace_obj ) , new_mc_obj ) LOGGER . debug ( 'Previous KL: Laplace: %f MC: %f' , np . sum ( prev_laplace_kl ) , np . sum ( prev_mc_kl ) ) LOGGER . debug ( 'Predicted new KL: Laplace: %f MC: %f' , np . sum ( new_pred_laplace_kl ) , np . sum ( new_mc_kl ) ) LOGGER . debug ( 'Actual new KL: Laplace: %f MC: %f' , np . sum ( new_actual_laplace_kl ) , np . sum ( new_mc_kl ) ) LOGGER . debug ( 'Previous w KL: Laplace: %f MC: %f' , prev_laplace_kl_sum , prev_mc_kl_sum ) LOGGER . debug ( 'Predicted w new KL: Laplace: %f MC: %f' , new_pred_laplace_kl_sum , new_mc_kl_sum ) LOGGER . debug ( 'Actual w new KL: Laplace %f MC: %f' , new_actual_laplace_kl_sum , new_mc_kl_sum ) LOGGER . debug ( 'Predicted/actual improvement: %f / %f' , predicted_impr , actual_impr ) # Compute actual KL step taken at last iteration. actual_step = self . cur [ m ] . traj_info . last_kl_step / ( self . _hyperparams [ 'kl_step' ] * self . T ) if actual_step < self . cur [ m ] . step_mult : self . cur [ m ] . step_mult = max ( actual_step , self . _hyperparams [ 'min_step_mult' ] ) self . _set_new_mult ( predicted_impr , actual_impr , m )