(self)
Backpropagate prediction error from a randomly sampled experience batch.

    A minibatch of experiences is randomly sampled from the model's experience
    replay buffer and used to update the weights of the q_network and
    target_q_network.
Backpropagate prediction error from a randomly sampled experience batch.
Backpropagate prediction error from a randomly sampled experience batch .
def training_step(self):
    """Backpropagate prediction error from a randomly sampled experience batch.

    A minibatch of experiences is randomly sampled from the model's experience
    replay buffer and used to update the weights of the q_network and
    target_q_network.
    """
    if self.num_times_train_called % self.dqn_hparams.train_every_nth == 0:
      if len(self.experience) < self.dqn_hparams.minibatch_size:
        return

      # Sample experience.
      samples = random.sample(range(len(self.experience)),
                              self.dqn_hparams.minibatch_size)
      samples = [self.experience[i] for i in samples]

      # Batch states.
      states = np.empty((len(samples), self.q_network.cell.state_size))
      new_states = np.empty((len(samples),
                             self.target_q_network.cell.state_size))
      reward_new_states = np.empty((len(samples),
                                    self.reward_rnn.cell.state_size))
      observations = np.empty((len(samples), self.input_size))
      new_observations = np.empty((len(samples), self.input_size))
      action_mask = np.zeros((len(samples), self.num_actions))
      rewards = np.empty((len(samples),))
      lengths = np.full(len(samples), 1, dtype=int)

      for i, (o, s, a, r, new_o, new_s, reward_s) in enumerate(samples):
        observations[i, :] = o
        new_observations[i, :] = new_o
        states[i, :] = s
        new_states[i, :] = new_s
        action_mask[i, :] = a
        rewards[i] = r
        reward_new_states[i, :] = reward_s

      observations = np.reshape(observations,
                                (len(samples), 1, self.input_size))
      new_observations = np.reshape(new_observations,
                                    (len(samples), 1, self.input_size))

      calc_summaries = self.iteration % 100 == 0
      calc_summaries = calc_summaries and self.summary_writer is not None

      if self.algorithm == 'g':
        _, _, target_vals, summary_str = self.session.run([
            self.prediction_error,
            self.train_op,
            self.target_vals,
            self.summarize if calc_summaries else self.no_op1,
        ], {
            self.reward_rnn.melody_sequence: new_observations,
            self.reward_rnn.initial_state: reward_new_states,
            self.reward_rnn.lengths: lengths,
            self.q_network.melody_sequence: observations,
            self.q_network.initial_state: states,
            self.q_network.lengths: lengths,
            self.target_q_network.melody_sequence: new_observations,
            self.target_q_network.initial_state: new_states,
            self.target_q_network.lengths: lengths,
            self.action_mask: action_mask,
            self.rewards: rewards,
        })
      else:
        _, _, target_vals, summary_str = self.session.run([
            self.prediction_error,
            self.train_op,
            self.target_vals,
            self.summarize if calc_summaries else self.no_op1,
        ], {
            self.q_network.melody_sequence: observations,
            self.q_network.initial_state: states,
            self.q_network.lengths: lengths,
            self.target_q_network.melody_sequence: new_observations,
            self.target_q_network.initial_state: new_states,
            self.target_q_network.lengths: lengths,
            self.action_mask: action_mask,
            self.rewards: rewards,
        })

      total_logs = (self.iteration * self.dqn_hparams.train_every_nth)
      if total_logs % self.output_every_nth == 0:
        self.target_val_list.append(np.mean(target_vals))

      self.session.run(self.target_network_update)

      if calc_summaries:
        self.summary_writer.add_summary(summary_str, self.iteration)

      self.iteration += 1

    self.num_times_train_called += 1
def training_step ( self ) : if self . num_times_train_called % self . dqn_hparams . train_every_nth == 0 : if len ( self . experience ) < self . dqn_hparams . minibatch_size : return # Sample experience. samples = random . sample ( range ( len ( self . experience ) ) , self . dqn_hparams . minibatch_size ) samples = [ self . experience [ i ] for i in samples ] # Batch states. states = np . empty ( ( len ( samples ) , self . q_network . cell . state_size ) ) new_states = np . empty ( ( len ( samples ) , self . target_q_network . cell . state_size ) ) reward_new_states = np . empty ( ( len ( samples ) , self . reward_rnn . cell . state_size ) ) observations = np . empty ( ( len ( samples ) , self . input_size ) ) new_observations = np . empty ( ( len ( samples ) , self . input_size ) ) action_mask = np . zeros ( ( len ( samples ) , self . num_actions ) ) rewards = np . empty ( ( len ( samples ) , ) ) lengths = np . full ( len ( samples ) , 1 , dtype = int ) for i , ( o , s , a , r , new_o , new_s , reward_s ) in enumerate ( samples ) : observations [ i , : ] = o new_observations [ i , : ] = new_o states [ i , : ] = s new_states [ i , : ] = new_s action_mask [ i , : ] = a rewards [ i ] = r reward_new_states [ i , : ] = reward_s observations = np . reshape ( observations , ( len ( samples ) , 1 , self . input_size ) ) new_observations = np . reshape ( new_observations , ( len ( samples ) , 1 , self . input_size ) ) calc_summaries = self . iteration % 100 == 0 calc_summaries = calc_summaries and self . summary_writer is not None if self . algorithm == 'g' : _ , _ , target_vals , summary_str = self . session . run ( [ self . prediction_error , self . train_op , self . target_vals , self . summarize if calc_summaries else self . no_op1 , ] , { self . reward_rnn . melody_sequence : new_observations , self . reward_rnn . initial_state : reward_new_states , self . reward_rnn . lengths : lengths , self . q_network . melody_sequence : observations , self . q_network . initial_state : states , self . q_network . lengths : lengths , self . target_q_network . melody_sequence : new_observations , self . target_q_network . initial_state : new_states , self . target_q_network . lengths : lengths , self . action_mask : action_mask , self . rewards : rewards , } ) else : _ , _ , target_vals , summary_str = self . session . run ( [ self . prediction_error , self . train_op , self . target_vals , self . summarize if calc_summaries else self . no_op1 , ] , { self . q_network . melody_sequence : observations , self . q_network . initial_state : states , self . q_network . lengths : lengths , self . target_q_network . melody_sequence : new_observations , self . target_q_network . initial_state : new_states , self . target_q_network . lengths : lengths , self . action_mask : action_mask , self . rewards : rewards , } ) total_logs = ( self . iteration * self . dqn_hparams . train_every_nth ) if total_logs % self . output_every_nth == 0 : self . target_val_list . append ( np . mean ( target_vals ) ) self . session . run ( self . target_network_update ) if calc_summaries : self . summary_writer . add_summary ( summary_str , self . iteration ) self . iteration += 1 self . num_times_train_called += 1