code: @Override
      public Gradient[] calcGradient(IActorCritic iac, Stack<MiniTrans<Integer>> rewards) {
          MiniTrans<Integer> minTrans = rewards.pop();
  
          int size = rewards.size();
  
          //if recurrent then train as a time serie with a batch size of 1
          boolean recurrent = getAsyncGlobal().getCurrent().isRecurrent();
  
          int[] shape = getHistoryProcessor() == null ? mdp.getObservationSpace().getShape()
                          : getHistoryProcessor().getConf().getShape();
          int[] nshape = recurrent ? Learning.makeShape(1, shape, size)
                          : Learning.makeShape(size, shape);
  
          INDArray input = Nd4j.create(nshape);
          INDArray targets = recurrent ? Nd4j.create(1, 1, size) : Nd4j.create(size, 1);
          INDArray logSoftmax = recurrent ? Nd4j.zeros(1, mdp.getActionSpace().getSize(), size)
                          : Nd4j.zeros(size, mdp.getActionSpace().getSize());
  
          double r = minTrans.getReward();
          for (int i = size - 1; i >= 0; i--) {
              minTrans = rewards.pop();
  
              r = minTrans.getReward() + conf.getGamma() * r;
              if (recurrent) {
                  input.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.point(i)).assign(minTrans.getObs());
              } else {
                  input.putRow(i, minTrans.getObs());
              }
  
              //the critic
              targets.putScalar(i, r);
  
              //the actor
              double expectedV = minTrans.getOutput()[0].getDouble(0);
              double advantage = r - expectedV;
              if (recurrent) {
                  logSoftmax.putScalar(0, minTrans.getAction(), i, advantage);
              } else {
                  logSoftmax.putScalar(i, minTrans.getAction(), advantage);
              }
          }
  
          return iac.gradient(input, new INDArray[] {targets, logSoftmax});
      }
docs: calc the gradients based on the n-step rewards