code: def define_ppo_step(data_points, hparams, action_space, lr):
    
    observation, action, discounted_reward, norm_advantage, old_pdf = data_points
  
    obs_shape = common_layers.shape_list(observation)
    observation = tf.reshape(
        observation, [obs_shape[0] * obs_shape[1]] + obs_shape[2:]
    )
    (logits, new_value) = get_policy(observation, hparams, action_space)
    logits = tf.reshape(logits, obs_shape[:2] + [action_space.n])
    new_value = tf.reshape(new_value, obs_shape[:2])
    new_policy_dist = tfp.distributions.Categorical(logits=logits)
  
    new_pdf = new_policy_dist.prob(action)
  
    ratio = new_pdf / old_pdf
    clipped_ratio = tf.clip_by_value(ratio, 1 - hparams.clipping_coef,
                                     1 + hparams.clipping_coef)
  
    surrogate_objective = tf.minimum(clipped_ratio * norm_advantage,
                                     ratio * norm_advantage)
    policy_loss = -tf.reduce_mean(surrogate_objective)
  
    value_error = new_value - discounted_reward
    value_loss = hparams.value_loss_coef * tf.reduce_mean(value_error ** 2)
  
    entropy = new_policy_dist.entropy()
    entropy_loss = -hparams.entropy_loss_coef * tf.reduce_mean(entropy)
  
    losses = [policy_loss, value_loss, entropy_loss]
    loss = sum(losses)
    variables = tf.global_variables(hparams.policy_network + "/.*")
    train_op = optimize.optimize(loss, lr, hparams, variables=variables)
  
    with tf.control_dependencies([train_op]):
      return [tf.identity(x) for x in losses]
docs: Define ppo step.