(r_N_T_D, gamma)
return q_N_T_D
Computes Q values from rewards.
    q_N_T_D[i,t,:] == r_N_T_D[i,t,:] + gamma*r_N_T_D[i,t+1,:] + gamma^2*r_N_T_D[i,t+2,:] + ...
Computes Q values from rewards.
    q_N_T_D[i,t,:] == r_N_T_D[i,t,:] + gamma*r_N_T_D[i,t+1,:] + gamma^2*r_N_T_D[i,t+2,:] + ...
Computes Q values from rewards . q_N_T_D [ i t : ] == r_N_T_D [ i t : ] + gamma * r_N_T_D [ i t + 1 : ] + gamma^2 * r_N_T_D [ i t + 2 : ] + ...
def discount(r_N_T_D, gamma):
    '''
    Computes Q values from rewards.
    q_N_T_D[i,t,:] == r_N_T_D[i,t,:] + gamma*r_N_T_D[i,t+1,:] + gamma^2*r_N_T_D[i,t+2,:] + ...
    '''
    assert r_N_T_D.ndim == 2 or r_N_T_D.ndim == 3
    input_ndim = r_N_T_D.ndim
    if r_N_T_D.ndim == 2: r_N_T_D = r_N_T_D[...,None]

    discfactors_T = np.power(gamma, np.arange(r_N_T_D.shape[1]))
    discounted_N_T_D = r_N_T_D * discfactors_T[None,:,None]
    q_N_T_D = np.cumsum(discounted_N_T_D[:,::-1,:], axis=1)[:,::-1,:] # this is equal to gamma**t * (r_N_T_D[i,t,:] + gamma*r_N_T_D[i,t+1,:] + ...)
    q_N_T_D /= discfactors_T[None,:,None]

    # Sanity check: Q values at last timestep should equal original rewards
    assert np.allclose(q_N_T_D[:,-1,:], r_N_T_D[:,-1,:])

    if input_ndim == 2:
        assert q_N_T_D.shape[-1] == 1
        return q_N_T_D[:,:,0]
    return q_N_T_D
def discount ( r_N_T_D , gamma ) : assert r_N_T_D . ndim == 2 or r_N_T_D . ndim == 3 input_ndim = r_N_T_D . ndim if r_N_T_D . ndim == 2 : r_N_T_D = r_N_T_D [ ... , None ] discfactors_T = np . power ( gamma , np . arange ( r_N_T_D . shape [ 1 ] ) ) discounted_N_T_D = r_N_T_D * discfactors_T [ None , : , None ] q_N_T_D = np . cumsum ( discounted_N_T_D [ : , : : - 1 , : ] , axis = 1 ) [ : , : : - 1 , : ] # this is equal to gamma**t * (r_N_T_D[i,t,:] + gamma*r_N_T_D[i,t+1,:] + ...) q_N_T_D /= discfactors_T [ None , : , None ] # Sanity check: Q values at last timestep should equal original rewards assert np . allclose ( q_N_T_D [ : , - 1 , : ] , r_N_T_D [ : , - 1 , : ] ) if input_ndim == 2 : assert q_N_T_D . shape [ - 1 ] == 1 return q_N_T_D [ : , : , 0 ] return q_N_T_D