code: def calculate_gradient(self, batch_info, device, model, rollout):
          
          evaluator = model.evaluate(rollout)
  
          # Use evaluator interface to get the what we are interested in from the model
          advantages = evaluator.get('rollout:advantages')
          returns = evaluator.get('rollout:returns')
          rollout_values = evaluator.get('rollout:values')
  
          logprobs = evaluator.get('model:action:logprobs')
          values = evaluator.get('model:values')
          entropy = evaluator.get('model:entropy')
  
          # Actual calculations. Pretty trivial
          policy_loss = -torch.mean(advantages * logprobs)
          value_loss = 0.5 * F.mse_loss(values, returns)
          policy_entropy = torch.mean(entropy)
  
          loss_value = (
              policy_loss - self.entropy_coefficient * policy_entropy + self.value_coefficient * value_loss
          )
  
          loss_value.backward()
  
          return {
              'policy_loss': policy_loss.item(),
              'value_loss': value_loss.item(),
              'policy_entropy': policy_entropy.item(),
              'advantage_norm': torch.norm(advantages).item(),
              'explained_variance': explained_variance(returns, rollout_values)
          }
docs: Calculate loss of the supplied rollout