(self, m, step=False)
Update the dual variables for the specified condition.
        Args:
            m: Condition
            step: Whether or not to update pol_wt.
Update the dual variables for the specified condition.
        Args:
            m: Condition
            step: Whether or not to update pol_wt.
Update the dual variables for the specified condition . Args : m : Condition step : Whether or not to update pol_wt .
def _policy_dual_step(self, m, step=False):
        """
        Update the dual variables for the specified condition.
        Args:
            m: Condition
            step: Whether or not to update pol_wt.
        """
        dU, T = self.dU, self.T
        samples = self.cur[m].sample_list
        N = len(samples)
        X = samples.get_X()
        if 'new_traj_distr' in dir(self):
            traj, pol_info = self.new_traj_distr[m], self.cur[m].pol_info
        else:
            traj, pol_info = self.cur[m].traj_distr, self.cur[m].pol_info

        # Compute trajectory action at each sampled state.
        traj_mu = np.zeros((N, T, dU))
        for i in range(N):
            for t in range(T):
                traj_mu[i, t, :] = traj.K[t, :, :].dot(X[i, t, :]) + \
                        traj.k[t, :]
        # Compute policy action at each sampled state.
        pol_mu = pol_info.pol_mu
        # Compute the difference and increment based on pol_wt.
        for t in range(T):
            tU, pU = traj_mu[:, t, :], pol_mu[:, t, :]
            # Increment mean term.
            pol_info.lambda_k[t, :] -= self._hyperparams['policy_dual_rate'] * \
                    pol_info.pol_wt[t] * \
                    traj.inv_pol_covar[t, :, :].dot(np.mean(tU - pU, axis=0))
            # Increment covariance term.
            t_covar, p_covar = traj.K[t, :, :], pol_info.pol_K[t, :, :]
            pol_info.lambda_K[t, :, :] -= \
                    self._hyperparams['policy_dual_rate_covar'] * \
                    pol_info.pol_wt[t] * \
                    traj.inv_pol_covar[t, :, :].dot(t_covar - p_covar)
        # Compute KL divergence.
        kl_m = self._policy_kl(m)[0]
        if step:
            lg_step = self._hyperparams['lg_step']
            # Increment pol_wt based on change in KL divergence.
            if self._hyperparams['fixed_lg_step'] == 1:
                # Take fixed size step.
                pol_info.pol_wt = np.array([
                    max(wt + lg_step, 0) for wt in pol_info.pol_wt
                ])
            elif self._hyperparams['fixed_lg_step'] == 2:
                # (In/De)crease based on change in constraint
                # satisfaction.
                if hasattr(pol_info, 'prev_kl'):
                    kl_change = kl_m / pol_info.prev_kl
                    for i in range(len(pol_info.pol_wt)):
                        if kl_change[i] < 0.8:
                            pol_info.pol_wt[i] *= 0.5
                        elif kl_change[i] >= 0.95:
                            pol_info.pol_wt[i] *= 2.0
            elif self._hyperparams['fixed_lg_step'] == 3:
                # (In/De)crease based on difference from average.
                if hasattr(pol_info, 'prev_kl'):
                    lower = np.mean(kl_m) - \
                            self._hyperparams['exp_step_lower'] * np.std(kl_m)
                    upper = np.mean(kl_m) + \
                            self._hyperparams['exp_step_upper'] * np.std(kl_m)
                    for i in range(len(pol_info.pol_wt)):
                        if kl_m[i] < lower:
                            pol_info.pol_wt[i] *= \
                                    self._hyperparams['exp_step_decrease']
                        elif kl_m[i] >= upper:
                            pol_info.pol_wt[i] *= \
                                    self._hyperparams['exp_step_increase']
            else:
                # Standard DGD step.
                pol_info.pol_wt = np.array([
                    max(pol_info.pol_wt[t] + lg_step * kl_m[t], 0)
                    for t in range(T)
                ])
            pol_info.prev_kl = kl_m
def _policy_dual_step ( self , m , step = False ) : dU , T = self . dU , self . T samples = self . cur [ m ] . sample_list N = len ( samples ) X = samples . get_X ( ) if 'new_traj_distr' in dir ( self ) : traj , pol_info = self . new_traj_distr [ m ] , self . cur [ m ] . pol_info else : traj , pol_info = self . cur [ m ] . traj_distr , self . cur [ m ] . pol_info # Compute trajectory action at each sampled state. traj_mu = np . zeros ( ( N , T , dU ) ) for i in range ( N ) : for t in range ( T ) : traj_mu [ i , t , : ] = traj . K [ t , : , : ] . dot ( X [ i , t , : ] ) + traj . k [ t , : ] # Compute policy action at each sampled state. pol_mu = pol_info . pol_mu # Compute the difference and increment based on pol_wt. for t in range ( T ) : tU , pU = traj_mu [ : , t , : ] , pol_mu [ : , t , : ] # Increment mean term. pol_info . lambda_k [ t , : ] -= self . _hyperparams [ 'policy_dual_rate' ] * pol_info . pol_wt [ t ] * traj . inv_pol_covar [ t , : , : ] . dot ( np . mean ( tU - pU , axis = 0 ) ) # Increment covariance term. t_covar , p_covar = traj . K [ t , : , : ] , pol_info . pol_K [ t , : , : ] pol_info . lambda_K [ t , : , : ] -= self . _hyperparams [ 'policy_dual_rate_covar' ] * pol_info . pol_wt [ t ] * traj . inv_pol_covar [ t , : , : ] . dot ( t_covar - p_covar ) # Compute KL divergence. kl_m = self . _policy_kl ( m ) [ 0 ] if step : lg_step = self . _hyperparams [ 'lg_step' ] # Increment pol_wt based on change in KL divergence. if self . _hyperparams [ 'fixed_lg_step' ] == 1 : # Take fixed size step. pol_info . pol_wt = np . array ( [ max ( wt + lg_step , 0 ) for wt in pol_info . pol_wt ] ) elif self . _hyperparams [ 'fixed_lg_step' ] == 2 : # (In/De)crease based on change in constraint # satisfaction. if hasattr ( pol_info , 'prev_kl' ) : kl_change = kl_m / pol_info . prev_kl for i in range ( len ( pol_info . pol_wt ) ) : if kl_change [ i ] < 0.8 : pol_info . pol_wt [ i ] *= 0.5 elif kl_change [ i ] >= 0.95 : pol_info . pol_wt [ i ] *= 2.0 elif self . _hyperparams [ 'fixed_lg_step' ] == 3 : # (In/De)crease based on difference from average. if hasattr ( pol_info , 'prev_kl' ) : lower = np . mean ( kl_m ) - self . _hyperparams [ 'exp_step_lower' ] * np . std ( kl_m ) upper = np . mean ( kl_m ) + self . _hyperparams [ 'exp_step_upper' ] * np . std ( kl_m ) for i in range ( len ( pol_info . pol_wt ) ) : if kl_m [ i ] < lower : pol_info . pol_wt [ i ] *= self . _hyperparams [ 'exp_step_decrease' ] elif kl_m [ i ] >= upper : pol_info . pol_wt [ i ] *= self . _hyperparams [ 'exp_step_increase' ] else : # Standard DGD step. pol_info . pol_wt = np . array ( [ max ( pol_info . pol_wt [ t ] + lg_step * kl_m [ t ] , 0 ) for t in range ( T ) ] ) pol_info . prev_kl = kl_m