(rollout, gamma, lambda_=1.0)
return Batch(batch_si, batch_a, batch_adv, batch_r, features, batch_c, batch_z)
given a rollout, compute its returns and the advantage
given a rollout, compute its returns and the advantage
given a rollout compute its returns and the advantage
def multiple_process_rollout(rollout, gamma, lambda_=1.0):
    """
    given a rollout, compute its returns and the advantage
    """
    batch_si = np.asarray(rollout['states'])
    batch_a = np.asarray(rollout['actions'])
    rewards = np.asarray(rollout['rewards'])
    vpred_t = np.hstack(
            [rollout['values'][:,:,0], np.expand_dims(rollout['r'], -1)])

    rewards_plus_v = np.hstack(
            [rollout['rewards'], np.expand_dims(rollout['r'], -1)])
    batch_r = discount(rewards_plus_v, gamma)[:,:-1]
    delta_t = rewards + gamma * vpred_t[:,1:] - vpred_t[:,:-1]

    batch_adv = discount(delta_t, gamma * lambda_)

    features = rollout['features'][:,0]

    if 'conditions' in rollout:
        batch_c = np.asarray(rollout['conditions'])
        batch_z = None
    else:
        batch_c = None
        batch_z = np.asarray(rollout['z'])

    #batch_a = flatten_first_two(batch_a)
    #batch_r = flatten_first_two(batch_r)
    #batch_si = flatten_first_two(batch_si)
    #batch_adv = flatten_first_two(batch_adv)
    #features = features[:,:,0,:]

    return Batch(batch_si, batch_a, batch_adv, batch_r, features, batch_c, batch_z)
def multiple_process_rollout ( rollout , gamma , lambda_ = 1.0 ) : batch_si = np . asarray ( rollout [ 'states' ] ) batch_a = np . asarray ( rollout [ 'actions' ] ) rewards = np . asarray ( rollout [ 'rewards' ] ) vpred_t = np . hstack ( [ rollout [ 'values' ] [ : , : , 0 ] , np . expand_dims ( rollout [ 'r' ] , - 1 ) ] ) rewards_plus_v = np . hstack ( [ rollout [ 'rewards' ] , np . expand_dims ( rollout [ 'r' ] , - 1 ) ] ) batch_r = discount ( rewards_plus_v , gamma ) [ : , : - 1 ] delta_t = rewards + gamma * vpred_t [ : , 1 : ] - vpred_t [ : , : - 1 ] batch_adv = discount ( delta_t , gamma * lambda_ ) features = rollout [ 'features' ] [ : , 0 ] if 'conditions' in rollout : batch_c = np . asarray ( rollout [ 'conditions' ] ) batch_z = None else : batch_c = None batch_z = np . asarray ( rollout [ 'z' ] ) #batch_a = flatten_first_two(batch_a) #batch_r = flatten_first_two(batch_r) #batch_si = flatten_first_two(batch_si) #batch_adv = flatten_first_two(batch_adv) #features = features[:,:,0,:] return Batch ( batch_si , batch_a , batch_adv , batch_r , features , batch_c , batch_z )