(self, policy_fn, obsfeat_fn, cfg, num_threads=None, no_reward=False)
return TrajBatch.FromTrajs(completed_trajs)
Run many simulations, with policy evaluations batched together.

        Samples complete trajectories (stopping when Simulation.done is true,
        or when cfg.max_traj_len is reached) until both
            (1) at least cfg.min_num_trajs trajectories have been sampled, and
            (2) at least cfg.min_total_sa transitions have been sampled.
Run many simulations, with policy evaluations batched together.
Run many simulations with policy evaluations batched together .
def sim_multi(self, policy_fn, obsfeat_fn, cfg, num_threads=None, no_reward=False):
        '''
        Run many simulations, with policy evaluations batched together.

        Samples complete trajectories (stopping when Simulation.done is true,
        or when cfg.max_traj_len is reached) until both
            (1) at least cfg.min_num_trajs trajectories have been sampled, and
            (2) at least cfg.min_total_sa transitions have been sampled.
        '''
        util.warn('sim_multi is deprecated!')
        assert isinstance(cfg, SimConfig)
        Do, Da = self.obs_space.storage_size, self.action_space.storage_size

        if num_threads is None:
            num_threads = multiprocessing.cpu_count()

        # Completed trajectories
        num_sa = 0
        completed_translists = []

        # Simulations and their current trajectories
        simbatch = self.new_batched_sim(cfg.batch_size) # TODO: reuse this across runs
        sim_trans_B = [[] for _ in xrange(cfg.batch_size)] # list of (o,obsfeat,adist,a,r) transitions for each simulation

        # Keep running simulations until we fill up the quota of trajectories and transitions
        while True:
            # If a simulation is done, pull out and save its trajectory, and restart it.
            for i_sim in xrange(cfg.batch_size):
                if simbatch.is_done(i_sim) or len(sim_trans_B[i_sim]) >= cfg.max_traj_len:
                    # Save the trajectory
                    completed_translists.append(sim_trans_B[i_sim])
                    num_sa += len(sim_trans_B[i_sim])
                    # and restart the simulation
                    sim_trans_B[i_sim] = []
                    simbatch.reset_sim(i_sim)

            # Are both quotas filled? If so, we're done.
            if len(completed_translists) >= cfg.min_num_trajs and num_sa >= cfg.min_total_sa:
                break

            # Keep simulating otherwise. Pull together observations from all simulations
            obs_B_Do = simbatch.batch_obs.copy(); assert obs_B_Do.shape == (cfg.batch_size, Do)

            # Evaluate policy
            obsfeat_B_Df = obsfeat_fn(obs_B_Do)
            a_B_Da, adist_B_Pa = policy_fn(obsfeat_B_Df)
            assert a_B_Da.shape == (cfg.batch_size, Da)
            assert adist_B_Pa.shape[0] == cfg.batch_size and adist_B_Pa.ndim == 2

            # Step simulations
            r_B = simbatch.batch_step(a_B_Da, num_threads=num_threads)
            if no_reward: r_B[:] = np.nan

            # Save the transitions
            for i_sim in xrange(cfg.batch_size):
                sim_trans_B[i_sim].append((obs_B_Do[i_sim,:], obsfeat_B_Df[i_sim,:], adist_B_Pa[i_sim,:], a_B_Da[i_sim,:], r_B[i_sim]))

        assert sum(len(tlist) for tlist in completed_translists) == num_sa

        # Pack together each trajectory individually
        def translist_to_traj(tlist):
            obs_T_Do = np.stack([trans[0] for trans in tlist]);  assert obs_T_Do.shape == (len(tlist), self.obs_space.storage_size)
            obsfeat_T_Df = np.stack([trans[1] for trans in tlist]); assert obsfeat_T_Df.shape[0] == len(tlist)
            adist_T_Pa = np.stack([trans[2] for trans in tlist]); assert adist_T_Pa.ndim == 2 and adist_T_Pa.shape[0] == len(tlist)
            a_T_Da = np.stack([trans[3] for trans in tlist]); assert a_T_Da.shape == (len(tlist), self.action_space.storage_size)
            r_T = np.stack([trans[4] for trans in tlist]); assert r_T.shape == (len(tlist),)
            return Trajectory(obs_T_Do, obsfeat_T_Df, adist_T_Pa, a_T_Da, r_T)
        completed_trajs = [translist_to_traj(tlist) for tlist in completed_translists]
        assert len(completed_trajs) >= cfg.min_num_trajs and sum(len(traj) for traj in completed_trajs) >= cfg.min_total_sa
        return TrajBatch.FromTrajs(completed_trajs)
def sim_multi ( self , policy_fn , obsfeat_fn , cfg , num_threads = None , no_reward = False ) : util . warn ( 'sim_multi is deprecated!' ) assert isinstance ( cfg , SimConfig ) Do , Da = self . obs_space . storage_size , self . action_space . storage_size if num_threads is None : num_threads = multiprocessing . cpu_count ( ) # Completed trajectories num_sa = 0 completed_translists = [ ] # Simulations and their current trajectories simbatch = self . new_batched_sim ( cfg . batch_size ) # TODO: reuse this across runs sim_trans_B = [ [ ] for _ in xrange ( cfg . batch_size ) ] # list of (o,obsfeat,adist,a,r) transitions for each simulation # Keep running simulations until we fill up the quota of trajectories and transitions while True : # If a simulation is done, pull out and save its trajectory, and restart it. for i_sim in xrange ( cfg . batch_size ) : if simbatch . is_done ( i_sim ) or len ( sim_trans_B [ i_sim ] ) >= cfg . max_traj_len : # Save the trajectory completed_translists . append ( sim_trans_B [ i_sim ] ) num_sa += len ( sim_trans_B [ i_sim ] ) # and restart the simulation sim_trans_B [ i_sim ] = [ ] simbatch . reset_sim ( i_sim ) # Are both quotas filled? If so, we're done. if len ( completed_translists ) >= cfg . min_num_trajs and num_sa >= cfg . min_total_sa : break # Keep simulating otherwise. Pull together observations from all simulations obs_B_Do = simbatch . batch_obs . copy ( ) assert obs_B_Do . shape == ( cfg . batch_size , Do ) # Evaluate policy obsfeat_B_Df = obsfeat_fn ( obs_B_Do ) a_B_Da , adist_B_Pa = policy_fn ( obsfeat_B_Df ) assert a_B_Da . shape == ( cfg . batch_size , Da ) assert adist_B_Pa . shape [ 0 ] == cfg . batch_size and adist_B_Pa . ndim == 2 # Step simulations r_B = simbatch . batch_step ( a_B_Da , num_threads = num_threads ) if no_reward : r_B [ : ] = np . nan # Save the transitions for i_sim in xrange ( cfg . batch_size ) : sim_trans_B [ i_sim ] . append ( ( obs_B_Do [ i_sim , : ] , obsfeat_B_Df [ i_sim , : ] , adist_B_Pa [ i_sim , : ] , a_B_Da [ i_sim , : ] , r_B [ i_sim ] ) ) assert sum ( len ( tlist ) for tlist in completed_translists ) == num_sa # Pack together each trajectory individually def translist_to_traj ( tlist ) : obs_T_Do = np . stack ( [ trans [ 0 ] for trans in tlist ] ) assert obs_T_Do . shape == ( len ( tlist ) , self . obs_space . storage_size ) obsfeat_T_Df = np . stack ( [ trans [ 1 ] for trans in tlist ] ) assert obsfeat_T_Df . shape [ 0 ] == len ( tlist ) adist_T_Pa = np . stack ( [ trans [ 2 ] for trans in tlist ] ) assert adist_T_Pa . ndim == 2 and adist_T_Pa . shape [ 0 ] == len ( tlist ) a_T_Da = np . stack ( [ trans [ 3 ] for trans in tlist ] ) assert a_T_Da . shape == ( len ( tlist ) , self . action_space . storage_size ) r_T = np . stack ( [ trans [ 4 ] for trans in tlist ] ) assert r_T . shape == ( len ( tlist ) , ) return Trajectory ( obs_T_Do , obsfeat_T_Df , adist_T_Pa , a_T_Da , r_T ) completed_trajs = [ translist_to_traj ( tlist ) for tlist in completed_translists ] assert len ( completed_trajs ) >= cfg . min_num_trajs and sum ( len ( traj ) for traj in completed_trajs ) >= cfg . min_total_sa return TrajBatch . FromTrajs ( completed_trajs )