code: def update(self, i):
          
          # Initialize gradients
          self.solver.zero_grad()
  
          # Forward and backward
          for _ in range(self.accum_grad):
              # feed data
              self.data_feeder()
  
              # forward
              for callback in self.forward_callback_on_finish:
                  callback(i)
              self.loss.forward(clear_no_need_grad=self.clear_buffer)
              for callback in self.forward_callback_on_finish:
                  callback(i)
  
              # backward
              for callback in self.backward_callback_on_start:
                  callback(i)
              self.loss.backward(clear_buffer=self.clear_buffer)
              for callback in self.backward_callback_on_finish:
                  callback(i)
  
          # AllReduce
          if self.comm and len(grads) != 0:
              for callback in self.comm_callback_on_start:
                  callback(i)
              self.comm.all_reduce(self.grads, division=False, inplace=False)
              for callback in self.comm_callback_on_finish:
                  callback(i)
  
          # Update
          for callback in self.update_callback_on_start:
              callback(i)
          self.solver.update()
          for callback in self.update_callback_on_finish:
              callback(i)
docs: Monolithic update method.
  
          This method calls the following methods with the dynamic loss scaling.
  
          1. solver.zerograd
          2. feed data
          3. loss.forward
          4. loss.backward
          5. comm.all_reduce (if it is specified)
          6. solver.update