(self, m, prev=False)
return kl_m, kl, kl_lm, kl_l
Monte-Carlo estimate of KL divergence between policy and
        trajectory.
Monte-Carlo estimate of KL divergence between policy and
        trajectory.
Monte - Carlo estimate of KL divergence between policy and trajectory .
def _policy_kl(self, m, prev=False):
        """
        Monte-Carlo estimate of KL divergence between policy and
        trajectory.
        """
        dU, T = self.dU, self.T
        if prev:
            traj, pol_info = self.prev[m].traj_distr, self.cur[m].pol_info
            samples = self.prev[m].sample_list
        else:
            traj, pol_info = self.cur[m].traj_distr, self.cur[m].pol_info
            samples = self.cur[m].sample_list
        N = len(samples)
        X, obs = samples.get_X(), samples.get_obs()
        kl, kl_m = np.zeros((N, T)), np.zeros(T)
        kl_l, kl_lm = np.zeros((N, T)), np.zeros(T)
        # Compute policy mean and covariance at each sample.
        pol_mu, _, pol_prec, pol_det_sigma = self.policy_opt.prob(obs.copy())
        # Compute KL divergence.
        for t in range(T):
            # Compute trajectory action at sample.
            traj_mu = np.zeros((N, dU))
            for i in range(N):
                traj_mu[i, :] = traj.K[t, :, :].dot(X[i, t, :]) + traj.k[t, :]
            diff = pol_mu[:, t, :] - traj_mu
            tr_pp_ct = pol_prec[:, t, :, :] * traj.pol_covar[t, :, :]
            k_ln_det_ct = 0.5 * dU + np.sum(
                np.log(np.diag(traj.chol_pol_covar[t, :, :]))
            )
            ln_det_cp = np.log(pol_det_sigma[:, t])
            # IMPORTANT: Note that this assumes that pol_prec does not
            #            depend on state!!!!
            #            (Only the last term makes this assumption.)
            d_pp_d = np.sum(diff * (diff.dot(pol_prec[1, t, :, :])), axis=1)
            kl[:, t] = 0.5 * np.sum(np.sum(tr_pp_ct, axis=1), axis=1) - \
                    k_ln_det_ct + 0.5 * ln_det_cp + 0.5 * d_pp_d
            tr_pp_ct_m = np.mean(tr_pp_ct, axis=0)
            kl_m[t] = 0.5 * np.sum(np.sum(tr_pp_ct_m, axis=0), axis=0) - \
                    k_ln_det_ct + 0.5 * np.mean(ln_det_cp) + \
                    0.5 * np.mean(d_pp_d)
            # Compute trajectory action at sample with Lagrange
            # multiplier.
            traj_mu = np.zeros((N, dU))
            for i in range(N):
                traj_mu[i, :] = \
                        (traj.K[t, :, :] - pol_info.lambda_K[t, :, :]).dot(
                            X[i, t, :]
                        ) + (traj.k[t, :] - pol_info.lambda_k[t, :])
            # Compute KL divergence with Lagrange multiplier.
            diff_l = pol_mu[:, t, :] - traj_mu
            d_pp_d_l = np.sum(diff_l * (diff_l.dot(pol_prec[1, t, :, :])),
                              axis=1)
            kl_l[:, t] = 0.5 * np.sum(np.sum(tr_pp_ct, axis=1), axis=1) - \
                    k_ln_det_ct + 0.5 * ln_det_cp + 0.5 * d_pp_d_l
            kl_lm[t] = 0.5 * np.sum(np.sum(tr_pp_ct_m, axis=0), axis=0) - \
                    k_ln_det_ct + 0.5 * np.mean(ln_det_cp) + \
                    0.5 * np.mean(d_pp_d_l)
        return kl_m, kl, kl_lm, kl_l
def _policy_kl ( self , m , prev = False ) : dU , T = self . dU , self . T if prev : traj , pol_info = self . prev [ m ] . traj_distr , self . cur [ m ] . pol_info samples = self . prev [ m ] . sample_list else : traj , pol_info = self . cur [ m ] . traj_distr , self . cur [ m ] . pol_info samples = self . cur [ m ] . sample_list N = len ( samples ) X , obs = samples . get_X ( ) , samples . get_obs ( ) kl , kl_m = np . zeros ( ( N , T ) ) , np . zeros ( T ) kl_l , kl_lm = np . zeros ( ( N , T ) ) , np . zeros ( T ) # Compute policy mean and covariance at each sample. pol_mu , _ , pol_prec , pol_det_sigma = self . policy_opt . prob ( obs . copy ( ) ) # Compute KL divergence. for t in range ( T ) : # Compute trajectory action at sample. traj_mu = np . zeros ( ( N , dU ) ) for i in range ( N ) : traj_mu [ i , : ] = traj . K [ t , : , : ] . dot ( X [ i , t , : ] ) + traj . k [ t , : ] diff = pol_mu [ : , t , : ] - traj_mu tr_pp_ct = pol_prec [ : , t , : , : ] * traj . pol_covar [ t , : , : ] k_ln_det_ct = 0.5 * dU + np . sum ( np . log ( np . diag ( traj . chol_pol_covar [ t , : , : ] ) ) ) ln_det_cp = np . log ( pol_det_sigma [ : , t ] ) # IMPORTANT: Note that this assumes that pol_prec does not #            depend on state!!!! #            (Only the last term makes this assumption.) d_pp_d = np . sum ( diff * ( diff . dot ( pol_prec [ 1 , t , : , : ] ) ) , axis = 1 ) kl [ : , t ] = 0.5 * np . sum ( np . sum ( tr_pp_ct , axis = 1 ) , axis = 1 ) - k_ln_det_ct + 0.5 * ln_det_cp + 0.5 * d_pp_d tr_pp_ct_m = np . mean ( tr_pp_ct , axis = 0 ) kl_m [ t ] = 0.5 * np . sum ( np . sum ( tr_pp_ct_m , axis = 0 ) , axis = 0 ) - k_ln_det_ct + 0.5 * np . mean ( ln_det_cp ) + 0.5 * np . mean ( d_pp_d ) # Compute trajectory action at sample with Lagrange # multiplier. traj_mu = np . zeros ( ( N , dU ) ) for i in range ( N ) : traj_mu [ i , : ] = ( traj . K [ t , : , : ] - pol_info . lambda_K [ t , : , : ] ) . dot ( X [ i , t , : ] ) + ( traj . k [ t , : ] - pol_info . lambda_k [ t , : ] ) # Compute KL divergence with Lagrange multiplier. diff_l = pol_mu [ : , t , : ] - traj_mu d_pp_d_l = np . sum ( diff_l * ( diff_l . dot ( pol_prec [ 1 , t , : , : ] ) ) , axis = 1 ) kl_l [ : , t ] = 0.5 * np . sum ( np . sum ( tr_pp_ct , axis = 1 ) , axis = 1 ) - k_ln_det_ct + 0.5 * ln_det_cp + 0.5 * d_pp_d_l kl_lm [ t ] = 0.5 * np . sum ( np . sum ( tr_pp_ct_m , axis = 0 ) , axis = 0 ) - k_ln_det_ct + 0.5 * np . mean ( ln_det_cp ) + 0.5 * np . mean ( d_pp_d_l ) return kl_m , kl , kl_lm , kl_l