code: def _update_step(self, sequence):
      
      observ, action, old_policy_params, reward, advantage = sequence['sequence']
      length = sequence['length']
      old_policy = self._policy_type(**old_policy_params)
      value_loss, value_summary = self._value_loss(observ, reward, length)
      network = self._network(observ, length)
      policy_loss, policy_summary = self._policy_loss(
          old_policy, network.policy, action, advantage, length)
      network_loss = network.get('loss', 0.0)
      loss = policy_loss + value_loss + tf.reduce_mean(network_loss)
      gradients, variables = (
          zip(*self._optimizer.compute_gradients(loss)))
      optimize = self._optimizer.apply_gradients(
          zip(gradients, variables))
      summary = tf.summary.merge([
          value_summary, policy_summary,
          tf.summary.histogram('network_loss', network_loss),
          tf.summary.scalar('avg_network_loss', tf.reduce_mean(network_loss)),
          tf.summary.scalar('gradient_norm', tf.global_norm(gradients)),
          utility.gradient_summaries(zip(gradients, variables))])
      with tf.control_dependencies([optimize]):
        return [tf.identity(x) for x in (value_loss, policy_loss, summary)]
docs: Compute the current combined loss and perform a gradient update step.
  
      The sequences must be a dict containing the keys `length` and `sequence`,
      where the latter is a tuple containing observations, actions, parameters of
      the behavioral policy, rewards, and advantages.
  
      Args:
        sequence: Sequences of episodes or chunks of episodes.
  
      Returns:
        Tuple of value loss, policy loss, and summary tensor.