code: def _apply_gradients_and_copy(self, opt, raw_grad_list, ps_var_grads):
          
          # TODO do this for variables together?
          with tf.name_scope('apply_gradients'):
              var_update_ops = []
              for vid, (g, v) in enumerate(ps_var_grads):
                  # TODO do we put momentum variables into local or global?
                  apply_gradient_op = opt.apply_gradients([(g, v)])
                  barrier = self._add_sync_queues_and_barrier(
                      'param_update_barrier_{}'.format(vid), [apply_gradient_op])
                  with tf.control_dependencies([barrier]), \
                          tf.device(self.cpu_device):
                      updated_value = v.read_value()
                      for towerid in range(self.nr_gpu):
                          var_update_ops.append(
                              raw_grad_list[towerid][vid][1].assign(updated_value))
              return var_update_ops
docs: Apply averaged gradients to ps vars, and then copy the updated
          variables back to each tower.
  
          Args:
              raw_grad_list: Ngpu x Nvar x 2 gradient list from all towers
              ps_var_grads: Nvar x 2 (grad, ps_var)
  
          Returns:
              list of copy ops