(self, inner_itr)
Compute the new policy.
Compute the new policy.
Compute the new policy .
def _update_policy(self, inner_itr):
        """ Compute the new policy. """
        dU, dO, T = self.dU, self.dO, self.T
        # Compute target mean, cov, and weight for each sample.
        obs_data, tgt_mu = np.zeros((0, T, dO)), np.zeros((0, T, dU))
        tgt_prc, tgt_wt = np.zeros((0, T, dU, dU)), np.zeros((0, T))
        for m in range(self.M):
            samples = self.cur[m].sample_list
            X = samples.get_X()
            N = len(samples)
            if inner_itr > 0:
                traj, pol_info = self.new_traj_distr[m], self.cur[m].pol_info
            else:
                traj, pol_info = self.cur[m].traj_distr, self.cur[m].pol_info
            mu = np.zeros((N, T, dU))
            prc = np.zeros((N, T, dU, dU))
            wt = np.zeros((N, T))
            # Get time-indexed actions.
            for t in range(T):
                # Compute actions along this trajectory.
                prc[:, t, :, :] = np.tile(traj.inv_pol_covar[t, :, :],
                                          [N, 1, 1])
                for i in range(N):
                    mu[i, t, :] = \
                            (traj.K[t, :, :].dot(X[i, t, :]) + traj.k[t, :]) - \
                            np.linalg.solve(
                                prc[i, t, :, :] / pol_info.pol_wt[t],
                                pol_info.lambda_K[t, :, :].dot(X[i, t, :]) + \
                                        pol_info.lambda_k[t, :]
                            )
                wt[:, t].fill(pol_info.pol_wt[t])
            tgt_mu = np.concatenate((tgt_mu, mu))
            tgt_prc = np.concatenate((tgt_prc, prc))
            tgt_wt = np.concatenate((tgt_wt, wt))
            obs_data = np.concatenate((obs_data, samples.get_obs()))
        self.policy_opt.update(obs_data, tgt_mu, tgt_prc, tgt_wt)
def _update_policy ( self , inner_itr ) : dU , dO , T = self . dU , self . dO , self . T # Compute target mean, cov, and weight for each sample. obs_data , tgt_mu = np . zeros ( ( 0 , T , dO ) ) , np . zeros ( ( 0 , T , dU ) ) tgt_prc , tgt_wt = np . zeros ( ( 0 , T , dU , dU ) ) , np . zeros ( ( 0 , T ) ) for m in range ( self . M ) : samples = self . cur [ m ] . sample_list X = samples . get_X ( ) N = len ( samples ) if inner_itr > 0 : traj , pol_info = self . new_traj_distr [ m ] , self . cur [ m ] . pol_info else : traj , pol_info = self . cur [ m ] . traj_distr , self . cur [ m ] . pol_info mu = np . zeros ( ( N , T , dU ) ) prc = np . zeros ( ( N , T , dU , dU ) ) wt = np . zeros ( ( N , T ) ) # Get time-indexed actions. for t in range ( T ) : # Compute actions along this trajectory. prc [ : , t , : , : ] = np . tile ( traj . inv_pol_covar [ t , : , : ] , [ N , 1 , 1 ] ) for i in range ( N ) : mu [ i , t , : ] = ( traj . K [ t , : , : ] . dot ( X [ i , t , : ] ) + traj . k [ t , : ] ) - np . linalg . solve ( prc [ i , t , : , : ] / pol_info . pol_wt [ t ] , pol_info . lambda_K [ t , : , : ] . dot ( X [ i , t , : ] ) + pol_info . lambda_k [ t , : ] ) wt [ : , t ] . fill ( pol_info . pol_wt [ t ] ) tgt_mu = np . concatenate ( ( tgt_mu , mu ) ) tgt_prc = np . concatenate ( ( tgt_prc , prc ) ) tgt_wt = np . concatenate ( ( tgt_wt , wt ) ) obs_data = np . concatenate ( ( obs_data , samples . get_obs ( ) ) ) self . policy_opt . update ( obs_data , tgt_mu , tgt_prc , tgt_wt )