(self, output_dir,

               # Hyperparameters
               dqn_hparams=None,
               reward_mode='music_theory_all',
               reward_scaler=1.0,
               exploration_mode='egreedy',
               priming_mode='random_note',
               stochastic_observations=False,
               algorithm='q',

               # Trained Note RNN to load and tune
               note_rnn_checkpoint_dir=None,
               note_rnn_checkpoint_file=None,
               note_rnn_type='default',
               note_rnn_hparams=None,

               # Other music related settings.
               num_notes_in_melody=32,
               input_size=rl_tuner_ops.NUM_CLASSES,
               num_actions=rl_tuner_ops.NUM_CLASSES,
               midi_primer=None,

               # Logistics.
               save_name='rl_tuner.ckpt',
               output_every_nth=1000,
               training_file_list=None,
               summary_writer=None,
               initialize_immediately=True)
Initializes the MelodyQNetwork class.

    Args:
      output_dir: Where the model will save its compositions (midi files).
      dqn_hparams: A HParams object containing the hyperparameters of
        the DQN algorithm, including minibatch size, exploration probability,
        etc.
      reward_mode: Controls which reward function can be applied. There are
        several, including 'scale', which teaches the model to play a scale,
        and of course 'music_theory_all', which is a music-theory-based reward
        function composed of other functions.
      reward_scaler: Controls the emphasis placed on the music theory rewards.
        This value is the inverse of 'c' in the academic paper.
      exploration_mode: can be 'egreedy' which is an epsilon greedy policy, or
        it can be 'boltzmann', in which the model will sample from its output
        distribution to choose the next action.
      priming_mode: Each time the model begins a new composition, it is primed
        with either a random note ('random_note'), a random MIDI file from the
        training data ('random_midi'), or a particular MIDI file
        ('single_midi').
      stochastic_observations: If False, the note that the model chooses to
        play next (the argmax of its softmax probabilities) deterministically
        becomes the next note it will observe. If True, the next observation
        will be sampled from the model's softmax output.
      algorithm: can be 'default', 'psi', 'g' or 'pure_rl', for different
        learning algorithms
      note_rnn_checkpoint_dir: The directory from which the internal
        NoteRNNLoader will load its checkpointed LSTM.
      note_rnn_checkpoint_file: A checkpoint file to use in case one cannot be
        found in the note_rnn_checkpoint_dir.
      note_rnn_type: If 'default', will use the basic LSTM described in the
        research paper. If 'basic_rnn', will assume the checkpoint is from a
        Magenta basic_rnn model.
      note_rnn_hparams: A HParams object which defines the hyper parameters
        used to train the MelodyRNN model that will be loaded from a checkpoint.
      num_notes_in_melody: The length of a composition of the model
      input_size: the size of the one-hot vector encoding a note that is input
        to the model.
      num_actions: The size of the one-hot vector encoding a note that is
        output by the model.
      midi_primer: A midi file that can be used to prime the model if
        priming_mode is set to 'single_midi'.
      save_name: Name the model will use to save checkpoints.
      output_every_nth: How many training steps before the model will print
        an output saying the cumulative reward, and save a checkpoint.
      training_file_list: A list of paths to tfrecord files containing melody
        training data. This is necessary to use the 'random_midi' priming mode.
      summary_writer: A tf.summary.FileWriter used to log metrics.
      initialize_immediately: if True, the class will instantiate its component
        MelodyRNN networks and build the graph in the constructor.
Initializes the MelodyQNetwork class.
Initializes the MelodyQNetwork class .
def __init__(self, output_dir,

               # Hyperparameters
               dqn_hparams=None,
               reward_mode='music_theory_all',
               reward_scaler=1.0,
               exploration_mode='egreedy',
               priming_mode='random_note',
               stochastic_observations=False,
               algorithm='q',

               # Trained Note RNN to load and tune
               note_rnn_checkpoint_dir=None,
               note_rnn_checkpoint_file=None,
               note_rnn_type='default',
               note_rnn_hparams=None,

               # Other music related settings.
               num_notes_in_melody=32,
               input_size=rl_tuner_ops.NUM_CLASSES,
               num_actions=rl_tuner_ops.NUM_CLASSES,
               midi_primer=None,

               # Logistics.
               save_name='rl_tuner.ckpt',
               output_every_nth=1000,
               training_file_list=None,
               summary_writer=None,
               initialize_immediately=True):
    """Initializes the MelodyQNetwork class.

    Args:
      output_dir: Where the model will save its compositions (midi files).
      dqn_hparams: A HParams object containing the hyperparameters of
        the DQN algorithm, including minibatch size, exploration probability,
        etc.
      reward_mode: Controls which reward function can be applied. There are
        several, including 'scale', which teaches the model to play a scale,
        and of course 'music_theory_all', which is a music-theory-based reward
        function composed of other functions.
      reward_scaler: Controls the emphasis placed on the music theory rewards.
        This value is the inverse of 'c' in the academic paper.
      exploration_mode: can be 'egreedy' which is an epsilon greedy policy, or
        it can be 'boltzmann', in which the model will sample from its output
        distribution to choose the next action.
      priming_mode: Each time the model begins a new composition, it is primed
        with either a random note ('random_note'), a random MIDI file from the
        training data ('random_midi'), or a particular MIDI file
        ('single_midi').
      stochastic_observations: If False, the note that the model chooses to
        play next (the argmax of its softmax probabilities) deterministically
        becomes the next note it will observe. If True, the next observation
        will be sampled from the model's softmax output.
      algorithm: can be 'default', 'psi', 'g' or 'pure_rl', for different
        learning algorithms
      note_rnn_checkpoint_dir: The directory from which the internal
        NoteRNNLoader will load its checkpointed LSTM.
      note_rnn_checkpoint_file: A checkpoint file to use in case one cannot be
        found in the note_rnn_checkpoint_dir.
      note_rnn_type: If 'default', will use the basic LSTM described in the
        research paper. If 'basic_rnn', will assume the checkpoint is from a
        Magenta basic_rnn model.
      note_rnn_hparams: A HParams object which defines the hyper parameters
        used to train the MelodyRNN model that will be loaded from a checkpoint.
      num_notes_in_melody: The length of a composition of the model
      input_size: the size of the one-hot vector encoding a note that is input
        to the model.
      num_actions: The size of the one-hot vector encoding a note that is
        output by the model.
      midi_primer: A midi file that can be used to prime the model if
        priming_mode is set to 'single_midi'.
      save_name: Name the model will use to save checkpoints.
      output_every_nth: How many training steps before the model will print
        an output saying the cumulative reward, and save a checkpoint.
      training_file_list: A list of paths to tfrecord files containing melody
        training data. This is necessary to use the 'random_midi' priming mode.
      summary_writer: A tf.summary.FileWriter used to log metrics.
      initialize_immediately: if True, the class will instantiate its component
        MelodyRNN networks and build the graph in the constructor.
    """
    # Make graph.
    self.graph = tf.Graph()

    with self.graph.as_default():
      # Memorize arguments.
      self.input_size = input_size
      self.num_actions = num_actions
      self.output_every_nth = output_every_nth
      self.output_dir = output_dir
      self.save_path = os.path.join(output_dir, save_name)
      self.reward_scaler = reward_scaler
      self.reward_mode = reward_mode
      self.exploration_mode = exploration_mode
      self.num_notes_in_melody = num_notes_in_melody
      self.stochastic_observations = stochastic_observations
      self.algorithm = algorithm
      self.priming_mode = priming_mode
      self.midi_primer = midi_primer
      self.training_file_list = training_file_list
      self.note_rnn_checkpoint_dir = note_rnn_checkpoint_dir
      self.note_rnn_checkpoint_file = note_rnn_checkpoint_file
      self.note_rnn_hparams = note_rnn_hparams
      self.note_rnn_type = note_rnn_type

      if priming_mode == 'single_midi' and midi_primer is None:
        tf.logging.fatal('A midi primer file is required when using'
                         'the single_midi priming mode.')

      if note_rnn_checkpoint_dir is None or not note_rnn_checkpoint_dir:
        print('Retrieving checkpoint of Note RNN from Magenta download server.')
        urllib.request.urlretrieve(
            'http://download.magenta.tensorflow.org/models/'
            'rl_tuner_note_rnn.ckpt', 'note_rnn.ckpt')
        self.note_rnn_checkpoint_dir = os.getcwd()
        self.note_rnn_checkpoint_file = os.path.join(os.getcwd(),
                                                     'note_rnn.ckpt')

      if self.note_rnn_hparams is None:
        if self.note_rnn_type == 'basic_rnn':
          self.note_rnn_hparams = rl_tuner_ops.basic_rnn_hparams()
        else:
          self.note_rnn_hparams = rl_tuner_ops.default_hparams()

      if self.algorithm == 'g' or self.algorithm == 'pure_rl':
        self.reward_mode = 'music_theory_only'

      if dqn_hparams is None:
        self.dqn_hparams = rl_tuner_ops.default_dqn_hparams()
      else:
        self.dqn_hparams = dqn_hparams
      self.discount_rate = tf.constant(self.dqn_hparams.discount_rate)
      self.target_network_update_rate = tf.constant(
          self.dqn_hparams.target_network_update_rate)

      self.optimizer = tf.train.AdamOptimizer()

      # DQN state.
      self.actions_executed_so_far = 0
      self.experience = collections.deque(
          maxlen=self.dqn_hparams.max_experience)
      self.iteration = 0
      self.summary_writer = summary_writer
      self.num_times_store_called = 0
      self.num_times_train_called = 0

    # Stored reward metrics.
    self.reward_last_n = 0
    self.rewards_batched = []
    self.music_theory_reward_last_n = 0
    self.music_theory_rewards_batched = []
    self.note_rnn_reward_last_n = 0
    self.note_rnn_rewards_batched = []
    self.eval_avg_reward = []
    self.eval_avg_music_theory_reward = []
    self.eval_avg_note_rnn_reward = []
    self.target_val_list = []

    # Variables to keep track of characteristics of the current composition
    # TODO(natashajaques): Implement composition as a class to obtain data
    # encapsulation so that you can't accidentally change the leap direction.
    self.beat = 0
    self.composition = []
    self.composition_direction = 0
    self.leapt_from = None  # stores the note at which composition leapt
    self.steps_since_last_leap = 0

    if not os.path.exists(self.output_dir):
      os.makedirs(self.output_dir)

    if initialize_immediately:
      self.initialize_internal_models_graph_session()
def __init__ ( self , output_dir , # Hyperparameters dqn_hparams = None , reward_mode = 'music_theory_all' , reward_scaler = 1.0 , exploration_mode = 'egreedy' , priming_mode = 'random_note' , stochastic_observations = False , algorithm = 'q' , # Trained Note RNN to load and tune note_rnn_checkpoint_dir = None , note_rnn_checkpoint_file = None , note_rnn_type = 'default' , note_rnn_hparams = None , # Other music related settings. num_notes_in_melody = 32 , input_size = rl_tuner_ops . NUM_CLASSES , num_actions = rl_tuner_ops . NUM_CLASSES , midi_primer = None , # Logistics. save_name = 'rl_tuner.ckpt' , output_every_nth = 1000 , training_file_list = None , summary_writer = None , initialize_immediately = True ) : # Make graph. self . graph = tf . Graph ( ) with self . graph . as_default ( ) : # Memorize arguments. self . input_size = input_size self . num_actions = num_actions self . output_every_nth = output_every_nth self . output_dir = output_dir self . save_path = os . path . join ( output_dir , save_name ) self . reward_scaler = reward_scaler self . reward_mode = reward_mode self . exploration_mode = exploration_mode self . num_notes_in_melody = num_notes_in_melody self . stochastic_observations = stochastic_observations self . algorithm = algorithm self . priming_mode = priming_mode self . midi_primer = midi_primer self . training_file_list = training_file_list self . note_rnn_checkpoint_dir = note_rnn_checkpoint_dir self . note_rnn_checkpoint_file = note_rnn_checkpoint_file self . note_rnn_hparams = note_rnn_hparams self . note_rnn_type = note_rnn_type if priming_mode == 'single_midi' and midi_primer is None : tf . logging . fatal ( 'A midi primer file is required when using' 'the single_midi priming mode.' ) if note_rnn_checkpoint_dir is None or not note_rnn_checkpoint_dir : print ( 'Retrieving checkpoint of Note RNN from Magenta download server.' ) urllib . request . urlretrieve ( 'http://download.magenta.tensorflow.org/models/' 'rl_tuner_note_rnn.ckpt' , 'note_rnn.ckpt' ) self . note_rnn_checkpoint_dir = os . getcwd ( ) self . note_rnn_checkpoint_file = os . path . join ( os . getcwd ( ) , 'note_rnn.ckpt' ) if self . note_rnn_hparams is None : if self . note_rnn_type == 'basic_rnn' : self . note_rnn_hparams = rl_tuner_ops . basic_rnn_hparams ( ) else : self . note_rnn_hparams = rl_tuner_ops . default_hparams ( ) if self . algorithm == 'g' or self . algorithm == 'pure_rl' : self . reward_mode = 'music_theory_only' if dqn_hparams is None : self . dqn_hparams = rl_tuner_ops . default_dqn_hparams ( ) else : self . dqn_hparams = dqn_hparams self . discount_rate = tf . constant ( self . dqn_hparams . discount_rate ) self . target_network_update_rate = tf . constant ( self . dqn_hparams . target_network_update_rate ) self . optimizer = tf . train . AdamOptimizer ( ) # DQN state. self . actions_executed_so_far = 0 self . experience = collections . deque ( maxlen = self . dqn_hparams . max_experience ) self . iteration = 0 self . summary_writer = summary_writer self . num_times_store_called = 0 self . num_times_train_called = 0 # Stored reward metrics. self . reward_last_n = 0 self . rewards_batched = [ ] self . music_theory_reward_last_n = 0 self . music_theory_rewards_batched = [ ] self . note_rnn_reward_last_n = 0 self . note_rnn_rewards_batched = [ ] self . eval_avg_reward = [ ] self . eval_avg_music_theory_reward = [ ] self . eval_avg_note_rnn_reward = [ ] self . target_val_list = [ ] # Variables to keep track of characteristics of the current composition # TODO(natashajaques): Implement composition as a class to obtain data # encapsulation so that you can't accidentally change the leap direction. self . beat = 0 self . composition = [ ] self . composition_direction = 0 self . leapt_from = None # stores the note at which composition leapt self . steps_since_last_leap = 0 if not os . path . exists ( self . output_dir ) : os . makedirs ( self . output_dir ) if initialize_immediately : self . initialize_internal_models_graph_session ( )