(use_tpu, model_params, model_dir,
                        include_features_in_predictions, decode_keys,
                        train_init_checkpoint, train_warmup_steps)
return model_fn
Returns an estimator model function.
Returns an estimator model function.
Returns an estimator model function .
def _estimator_model_fn(use_tpu, model_params, model_dir,
                        include_features_in_predictions, decode_keys,
                        train_init_checkpoint, train_warmup_steps):
  """Returns an estimator model function."""

  def model_fn(features, labels, mode, config, params):
    """Estimator model function."""

    del labels
    del config
    del params

    tf.get_variable_scope().set_initializer(
        tf.variance_scaling_initializer(
            1.0, mode="fan_avg", distribution="uniform"))

    if mode == tf.estimator.ModeKeys.PREDICT:
      predictions = model_params.estimator_prediction_fn(features)

      if include_features_in_predictions:
        predictions.update(features)

      if decode_keys:
        # Decode the raw ids into strings in prediction.
        def decode_host_call(tensor_dict):
          for key in decode_keys:
            predictions[key] = public_parsing_ops.decode(
                tensor_dict[key], model_params.vocab_filename,
                model_params.encoder_type)
          return tensor_dict

        contrib_tpu.outside_compilation(decode_host_call, predictions)
      return tpu_estimator.TPUEstimatorSpec(mode=mode, predictions=predictions)

    training = mode == tf.estimator.ModeKeys.TRAIN
    if use_tpu and model_params.use_bfloat16:
      with contrib_tpu.bfloat16_scope():
        loss, outputs = model_params.model()(features, training)
    else:
      loss, outputs = model_params.model()(features, training)

    # TPU requires ouputs all have batch dimension and doesn't handle scalar.
    # Tile all scalars to 1 dimension vector.
    outputs = _tile_scalar_to_batch_size(outputs, model_params.batch_size)

    if mode == tf.estimator.ModeKeys.TRAIN:
      init_lr = model_params.learning_rate
      global_step = tf.train.get_global_step()
      lr = init_lr / 0.01 * tf.rsqrt(
          tf.maximum(tf.to_float(global_step), 10000))
      if train_init_checkpoint:
        lr = tf.minimum(
            tf.to_float(global_step + 1) / train_warmup_steps * init_lr, lr)

      optimizer = adafactor.AdafactorOptimizer(
          learning_rate=lr,
          decay_rate=adafactor.adafactor_decay_rate_pow(0.8),
          beta1=0.0)
      if use_tpu:
        optimizer = tpu_optimizer.CrossShardOptimizer(optimizer)
      train_op = optimizer.minimize(loss, global_step=global_step)

      return tpu_estimator.TPUEstimatorSpec(
          mode=mode,
          loss=loss,
          train_op=train_op,
          scaffold_fn=_load_vars_from_checkpoint(use_tpu,
                                                 train_init_checkpoint),
          host_call=add_scalars_to_summary(model_dir, {"learning_rate": lr}))
    if mode == tf.estimator.ModeKeys.EVAL:
      eval_metrics = model_params.estimator_eval_metrics_fn(features, outputs)
      return tpu_estimator.TPUEstimatorSpec(
          mode=mode, loss=loss, eval_metrics=eval_metrics)

  return model_fn
def _estimator_model_fn ( use_tpu , model_params , model_dir , include_features_in_predictions , decode_keys , train_init_checkpoint , train_warmup_steps ) : def model_fn ( features , labels , mode , config , params ) : """Estimator model function.""" del labels del config del params tf . get_variable_scope ( ) . set_initializer ( tf . variance_scaling_initializer ( 1.0 , mode = "fan_avg" , distribution = "uniform" ) ) if mode == tf . estimator . ModeKeys . PREDICT : predictions = model_params . estimator_prediction_fn ( features ) if include_features_in_predictions : predictions . update ( features ) if decode_keys : # Decode the raw ids into strings in prediction. def decode_host_call ( tensor_dict ) : for key in decode_keys : predictions [ key ] = public_parsing_ops . decode ( tensor_dict [ key ] , model_params . vocab_filename , model_params . encoder_type ) return tensor_dict contrib_tpu . outside_compilation ( decode_host_call , predictions ) return tpu_estimator . TPUEstimatorSpec ( mode = mode , predictions = predictions ) training = mode == tf . estimator . ModeKeys . TRAIN if use_tpu and model_params . use_bfloat16 : with contrib_tpu . bfloat16_scope ( ) : loss , outputs = model_params . model ( ) ( features , training ) else : loss , outputs = model_params . model ( ) ( features , training ) # TPU requires ouputs all have batch dimension and doesn't handle scalar. # Tile all scalars to 1 dimension vector. outputs = _tile_scalar_to_batch_size ( outputs , model_params . batch_size ) if mode == tf . estimator . ModeKeys . TRAIN : init_lr = model_params . learning_rate global_step = tf . train . get_global_step ( ) lr = init_lr / 0.01 * tf . rsqrt ( tf . maximum ( tf . to_float ( global_step ) , 10000 ) ) if train_init_checkpoint : lr = tf . minimum ( tf . to_float ( global_step + 1 ) / train_warmup_steps * init_lr , lr ) optimizer = adafactor . AdafactorOptimizer ( learning_rate = lr , decay_rate = adafactor . adafactor_decay_rate_pow ( 0.8 ) , beta1 = 0.0 ) if use_tpu : optimizer = tpu_optimizer . CrossShardOptimizer ( optimizer ) train_op = optimizer . minimize ( loss , global_step = global_step ) return tpu_estimator . TPUEstimatorSpec ( mode = mode , loss = loss , train_op = train_op , scaffold_fn = _load_vars_from_checkpoint ( use_tpu , train_init_checkpoint ) , host_call = add_scalars_to_summary ( model_dir , { "learning_rate" : lr } ) ) if mode == tf . estimator . ModeKeys . EVAL : eval_metrics = model_params . estimator_eval_metrics_fn ( features , outputs ) return tpu_estimator . TPUEstimatorSpec ( mode = mode , loss = loss , eval_metrics = eval_metrics ) return model_fn