(self)
return lr
Get the learning rate.
Get the learning rate.
Get the learning rate .
def get_learning_rate(self):
    """Get the learning rate."""
    lrconf = self.config['solver']['optimizer']['learning_rate']
    learning_rate = lrconf['rate']
    learning_type = lrconf['type']

    #pylint: disable=invalid-name
    if learning_type == 'exp_decay':
      lr = tf.train.exponential_decay(
          learning_rate,
          tf.train.get_or_create_global_step(),
          lrconf['decay_steps'],
          lrconf['decay_rate'],
          staircase=True)
    elif learning_type == 'piecewise':
      #boundaries = [15000, 30000]
      #values = [1e-3, 1e-4, 1e-5]
      boundaries = lrconf['boundaries']
      values = lrconf['values']
      assert len(values) == len(
          boundaries) + 1, 'values len must equal boundaries len plus one'
      lr = tf.train.piecewise_constant(
          tf.train.get_or_create_global_step(),
          boundaries=boundaries,
          values=values)
    elif learning_type == 'warmup':
      learning_rate = tf.constant(
          value=learning_rate, shape=[], dtype=tf.float32)
      global_step = tf.train.get_or_create_global_step()
      data_size = self.config['data']['train_data_size']
      num_epochs = self.config["data"]["task"]['epochs']
      batch_size = self.config["data"]["task"]['batch_size']
      num_batch = int(math.ceil(data_size * num_epochs / batch_size))
      learning_rate = tf.train.polynomial_decay(
          learning_rate,
          global_step,
          num_batch,
          end_learning_rate=0.0,
          power=1.0,
          cycle=False)
      global_steps_int = tf.cast(global_step, tf.int32)
      warmup_steps_int = tf.constant(lrconf['num_warmup_steps'], dtype=tf.int32)

      global_steps_float = tf.cast(global_steps_int, tf.float32)
      warmup_steps_float = tf.cast(warmup_steps_int, tf.float32)

      warmup_percent_done = global_steps_float / warmup_steps_float
      warmup_learning_rate = learning_rate * warmup_percent_done

      is_warmup = tf.cast(global_steps_int < warmup_steps_int, tf.float32)
      lr = ((1.0 - is_warmup) * learning_rate +
            is_warmup * warmup_learning_rate)
    elif learning_type == 'const':
      lr = learning_rate
    else:
      raise ValueError(
          "Not support learning rate type: {}".format(learning_type))
    tf.summary.scalar('lr', lr)
    return lr
def get_learning_rate ( self ) : lrconf = self . config [ 'solver' ] [ 'optimizer' ] [ 'learning_rate' ] learning_rate = lrconf [ 'rate' ] learning_type = lrconf [ 'type' ] #pylint: disable=invalid-name if learning_type == 'exp_decay' : lr = tf . train . exponential_decay ( learning_rate , tf . train . get_or_create_global_step ( ) , lrconf [ 'decay_steps' ] , lrconf [ 'decay_rate' ] , staircase = True ) elif learning_type == 'piecewise' : #boundaries = [15000, 30000] #values = [1e-3, 1e-4, 1e-5] boundaries = lrconf [ 'boundaries' ] values = lrconf [ 'values' ] assert len ( values ) == len ( boundaries ) + 1 , 'values len must equal boundaries len plus one' lr = tf . train . piecewise_constant ( tf . train . get_or_create_global_step ( ) , boundaries = boundaries , values = values ) elif learning_type == 'warmup' : learning_rate = tf . constant ( value = learning_rate , shape = [ ] , dtype = tf . float32 ) global_step = tf . train . get_or_create_global_step ( ) data_size = self . config [ 'data' ] [ 'train_data_size' ] num_epochs = self . config [ "data" ] [ "task" ] [ 'epochs' ] batch_size = self . config [ "data" ] [ "task" ] [ 'batch_size' ] num_batch = int ( math . ceil ( data_size * num_epochs / batch_size ) ) learning_rate = tf . train . polynomial_decay ( learning_rate , global_step , num_batch , end_learning_rate = 0.0 , power = 1.0 , cycle = False ) global_steps_int = tf . cast ( global_step , tf . int32 ) warmup_steps_int = tf . constant ( lrconf [ 'num_warmup_steps' ] , dtype = tf . int32 ) global_steps_float = tf . cast ( global_steps_int , tf . float32 ) warmup_steps_float = tf . cast ( warmup_steps_int , tf . float32 ) warmup_percent_done = global_steps_float / warmup_steps_float warmup_learning_rate = learning_rate * warmup_percent_done is_warmup = tf . cast ( global_steps_int < warmup_steps_int , tf . float32 ) lr = ( ( 1.0 - is_warmup ) * learning_rate + is_warmup * warmup_learning_rate ) elif learning_type == 'const' : lr = learning_rate else : raise ValueError ( "Not support learning rate type: {}" . format ( learning_type ) ) tf . summary . scalar ( 'lr' , lr ) return lr