(self, samples, costs, mean_old, cov_old,
                   fixed_eta=None, use_fixed_eta=False)
return mean_new, cov_new, inv_cov_new, chol_cov_new, etas
Perform optimization with PI2. Computes new mean and covariance matrices
        of the policy parameters given policy samples and their costs.
        Args:
            samples: Matrix of policy samples with dimensions: 
                     [num_samples x num_timesteps x num_controls].
            costs: Matrix of roll-out costs with dimensions:
                   [num_samples x num_timesteps]
            mean_old: Old policy mean.
            cov_old: Old policy covariance.            
            fixed_eta: Fixed value of eta to use if use_fixed_eta is True.
            use_fixed_eta: Whether to use fixed_eta or compute using KL dual.
        Returns:
            mean_new: New policy mean.
            cov_new: New policy covariance.
            inv_cov_new: Inverse of the new policy covariance.
            chol_cov_new: Cholesky decomposition of the new policy covariance.
Perform optimization with PI2. Computes new mean and covariance matrices
        of the policy parameters given policy samples and their costs.
        Args:
            samples: Matrix of policy samples with dimensions: 
                     [num_samples x num_timesteps x num_controls].
            costs: Matrix of roll-out costs with dimensions:
                   [num_samples x num_timesteps]
            mean_old: Old policy mean.
            cov_old: Old policy covariance.            
            fixed_eta: Fixed value of eta to use if use_fixed_eta is True.
            use_fixed_eta: Whether to use fixed_eta or compute using KL dual.
        Returns:
            mean_new: New policy mean.
            cov_new: New policy covariance.
            inv_cov_new: Inverse of the new policy covariance.
            chol_cov_new: Cholesky decomposition of the new policy covariance.
Perform optimization with PI2 . Computes new mean and covariance matrices of the policy parameters given policy samples and their costs . Args : samples : Matrix of policy samples with dimensions : [ num_samples x num_timesteps x num_controls ] . costs : Matrix of roll - out costs with dimensions : [ num_samples x num_timesteps ] mean_old : Old policy mean . cov_old : Old policy covariance . fixed_eta : Fixed value of eta to use if use_fixed_eta is True . use_fixed_eta : Whether to use fixed_eta or compute using KL dual . Returns : mean_new : New policy mean . cov_new : New policy covariance . inv_cov_new : Inverse of the new policy covariance . chol_cov_new : Cholesky decomposition of the new policy covariance .
def update_pi2(self, samples, costs, mean_old, cov_old,
                   fixed_eta=None, use_fixed_eta=False):
        """
        Perform optimization with PI2. Computes new mean and covariance matrices
        of the policy parameters given policy samples and their costs.
        Args:
            samples: Matrix of policy samples with dimensions: 
                     [num_samples x num_timesteps x num_controls].
            costs: Matrix of roll-out costs with dimensions:
                   [num_samples x num_timesteps]
            mean_old: Old policy mean.
            cov_old: Old policy covariance.            
            fixed_eta: Fixed value of eta to use if use_fixed_eta is True.
            use_fixed_eta: Whether to use fixed_eta or compute using KL dual.
        Returns:
            mean_new: New policy mean.
            cov_new: New policy covariance.
            inv_cov_new: Inverse of the new policy covariance.
            chol_cov_new: Cholesky decomposition of the new policy covariance.
        """
        mean_new = np.zeros(mean_old.shape)
        cov_new = np.zeros(cov_old.shape)
        inv_cov_new = np.zeros(cov_old.shape)
        chol_cov_new = np.zeros(cov_old.shape)

        # Iterate over time steps.
        T = samples.shape[1]
        etas = np.zeros(T)

        del_ = self._hyperparams['del0']
        if self._hyperparams['pi2_cons_per_step']:
            del_ = np.ones(T) * del_

        fail = True
        while fail:
            fail = False
            for t in xrange(T):

                # Compute cost-to-go for each time step for each sample.
                cost_to_go = np.sum(costs[:, t:T], axis=1)

                if use_fixed_eta:
                    eta = (fixed_eta[t] if isinstance(fixed_eta, np.ndarray)
                           else fixed_eta)
                else:
                    # Perform REPS-like optimization of the temperature eta.
                    res = minimize(self.kl_dual, 10.0,
                                   bounds=((self._min_temperature, None),),
                                   args=(self._kl_threshold, cost_to_go))
                    etas[t] = res.x
                    eta = res.x

                exponent = -cost_to_go

                exp_cost = np.exp((exponent - np.max(exponent)) / eta)
                prob = exp_cost / np.sum(exp_cost)

                # Update policy mean with weighted max-likelihood.
                mean_new[t] = np.sum(prob[:, np.newaxis] * samples[:, t], axis=0)

                # Update policy covariance with weighted max-likelihood.
                for i in xrange(samples.shape[0]):
                    mean_diff = samples[i, t] - mean_new[t]
                    mean_diff = np.reshape(mean_diff, (len(mean_diff), 1))
                    cov_new[t] += prob[i] * np.dot(mean_diff, mean_diff.T)

                # If covariance damping is enabled, compute covariance as multiple
                # of the old covariance. The multiplier is first fitted using
                # max-likelihood and then taken to the power (1/covariance_damping).
                if(self._covariance_damping is not None
                   and self._covariance_damping > 0.0):

                    mult = np.trace(np.dot(sp.linalg.inv(cov_old[t]),
                        cov_new[t])) / len(cov_old[t])
                    mult = np.power(mult, 1 / self._covariance_damping)
                    cov_new[t] = mult * cov_old[t]

                # Compute covariance inverse and cholesky decomposition.
                try:
                    inv_cov_new[t] = sp.linalg.inv(cov_new[t])
                    chol_cov_new[t] = sp.linalg.cholesky(cov_new[t])
                except LinAlgError:
                    fail = True
                    old_eta = eta
                    eta = fixed_eta if use_fixed_eta else etas
                    eta[t] += del_[t]
                    LOGGER.debug('Increasing eta %d: %f -> %f', t, old_eta, eta[t])
                    del_[t] *= 2
                    if eta[t] >= 1e16:
                        raise ValueError

        return mean_new, cov_new, inv_cov_new, chol_cov_new, etas
def update_pi2 ( self , samples , costs , mean_old , cov_old , fixed_eta = None , use_fixed_eta = False ) : mean_new = np . zeros ( mean_old . shape ) cov_new = np . zeros ( cov_old . shape ) inv_cov_new = np . zeros ( cov_old . shape ) chol_cov_new = np . zeros ( cov_old . shape ) # Iterate over time steps. T = samples . shape [ 1 ] etas = np . zeros ( T ) del_ = self . _hyperparams [ 'del0' ] if self . _hyperparams [ 'pi2_cons_per_step' ] : del_ = np . ones ( T ) * del_ fail = True while fail : fail = False for t in xrange ( T ) : # Compute cost-to-go for each time step for each sample. cost_to_go = np . sum ( costs [ : , t : T ] , axis = 1 ) if use_fixed_eta : eta = ( fixed_eta [ t ] if isinstance ( fixed_eta , np . ndarray ) else fixed_eta ) else : # Perform REPS-like optimization of the temperature eta. res = minimize ( self . kl_dual , 10.0 , bounds = ( ( self . _min_temperature , None ) , ) , args = ( self . _kl_threshold , cost_to_go ) ) etas [ t ] = res . x eta = res . x exponent = - cost_to_go exp_cost = np . exp ( ( exponent - np . max ( exponent ) ) / eta ) prob = exp_cost / np . sum ( exp_cost ) # Update policy mean with weighted max-likelihood. mean_new [ t ] = np . sum ( prob [ : , np . newaxis ] * samples [ : , t ] , axis = 0 ) # Update policy covariance with weighted max-likelihood. for i in xrange ( samples . shape [ 0 ] ) : mean_diff = samples [ i , t ] - mean_new [ t ] mean_diff = np . reshape ( mean_diff , ( len ( mean_diff ) , 1 ) ) cov_new [ t ] += prob [ i ] * np . dot ( mean_diff , mean_diff . T ) # If covariance damping is enabled, compute covariance as multiple # of the old covariance. The multiplier is first fitted using # max-likelihood and then taken to the power (1/covariance_damping). if ( self . _covariance_damping is not None and self . _covariance_damping > 0.0 ) : mult = np . trace ( np . dot ( sp . linalg . inv ( cov_old [ t ] ) , cov_new [ t ] ) ) / len ( cov_old [ t ] ) mult = np . power ( mult , 1 / self . _covariance_damping ) cov_new [ t ] = mult * cov_old [ t ] # Compute covariance inverse and cholesky decomposition. try : inv_cov_new [ t ] = sp . linalg . inv ( cov_new [ t ] ) chol_cov_new [ t ] = sp . linalg . cholesky ( cov_new [ t ] ) except LinAlgError : fail = True old_eta = eta eta = fixed_eta if use_fixed_eta else etas eta [ t ] += del_ [ t ] LOGGER . debug ( 'Increasing eta %d: %f -> %f' , t , old_eta , eta [ t ] ) del_ [ t ] *= 2 if eta [ t ] >= 1e16 : raise ValueError return mean_new , cov_new , inv_cov_new , chol_cov_new , etas