code: def step(self, exploration):
          
          old_s = self._current_ob
          if self.rng.rand() <= exploration:
              act = self.rng.choice(range(self.num_actions))
          else:
              history = self.recent_state()
              history.append(old_s)
              history = np.stack(history, axis=-1)  # state_shape + (Hist,)
  
              # assume batched network
              history = np.expand_dims(history, axis=0)
              q_values = self.predictor(history)[0][0]  # this is the bottleneck
              act = np.argmax(q_values)
  
          self._current_ob, reward, isOver, info = self.player.step(act)
          self._current_game_score.feed(reward)
          self._current_episode.append(Experience(old_s, act, reward, isOver))
  
          if isOver:
              flush_experience = True
              if 'ale.lives' in info:  # if running Atari, do something special
                  if info['ale.lives'] != 0:
                      # only record score and flush experience
                      # when a whole game is over (not when an episode is over)
                      flush_experience = False
              self.player.reset()
  
              if flush_experience:
                  self.total_scores.append(self._current_game_score.sum)
                  self._current_game_score.reset()
  
                  # Ensure that the whole episode of experience is continuous in the replay buffer
                  with self.memory.writer_lock:
                      for exp in self._current_episode:
                          self.memory.append(exp)
                  self._current_episode.clear()
docs: Run the environment for one step.
          If the episode ends, store the entire episode to the replay memory.