(FLAGS, features, is_training)
return return_dict
Loss for downstream span-extraction QA tasks such as SQuAD.
Loss for downstream span-extraction QA tasks such as SQuAD.
Loss for downstream span - extraction QA tasks such as SQuAD .
def get_qa_outputs(FLAGS, features, is_training):
  """Loss for downstream span-extraction QA tasks such as SQuAD."""

  inp = tf.transpose(features["input_ids"], [1, 0])
  seg_id = tf.transpose(features["segment_ids"], [1, 0])
  inp_mask = tf.transpose(features["input_mask"], [1, 0])

  seq_len = tf.shape(inp)[0]

  xlnet_config = xlnet.XLNetConfig(json_path=FLAGS.model_config_path)
  run_config = xlnet.create_run_config(is_training, True, FLAGS)

  xlnet_model = xlnet.XLNetModel(
      xlnet_config=xlnet_config,
      run_config=run_config,
      input_ids=inp,
      seg_ids=seg_id,
      input_mask=inp_mask)
  output = xlnet_model.get_sequence_output()
  initializer = xlnet_model.get_initializer()

  return_dict = {}

  # invalid position mask such as query and special symbols (PAD, SEP, CLS)
  p_mask = features["p_mask"]

  # logit of the start position
  with tf.variable_scope("start_logits"):
    start_logits = tf.layers.dense(
        output,
        1,
        kernel_initializer=initializer)
    start_logits = tf.transpose(tf.squeeze(start_logits, -1), [1, 0])
    start_logits_masked = start_logits * (1 - p_mask) - 1e30 * p_mask
    start_log_probs = tf.nn.log_softmax(start_logits_masked, -1)

  # logit of the end position
  with tf.variable_scope("end_logits"):
    if is_training:
      # during training, compute the end logits based on the
      # ground truth of the start position

      start_positions = tf.reshape(features["start_positions"], [-1])
      start_index = tf.one_hot(start_positions, depth=seq_len, axis=-1,
                               dtype=tf.float32)
      start_features = tf.einsum("lbh,bl->bh", output, start_index)
      start_features = tf.tile(start_features[None], [seq_len, 1, 1])
      end_logits = tf.layers.dense(
          tf.concat([output, start_features], axis=-1), xlnet_config.d_model,
          kernel_initializer=initializer, activation=tf.tanh, name="dense_0")
      end_logits = tf.contrib.layers.layer_norm(
          end_logits, begin_norm_axis=-1)

      end_logits = tf.layers.dense(
          end_logits, 1,
          kernel_initializer=initializer,
          name="dense_1")
      end_logits = tf.transpose(tf.squeeze(end_logits, -1), [1, 0])
      end_logits_masked = end_logits * (1 - p_mask) - 1e30 * p_mask
      end_log_probs = tf.nn.log_softmax(end_logits_masked, -1)
    else:
      # during inference, compute the end logits based on beam search

      start_top_log_probs, start_top_index = tf.nn.top_k(
          start_log_probs, k=FLAGS.start_n_top)
      start_index = tf.one_hot(start_top_index,
                               depth=seq_len, axis=-1, dtype=tf.float32)
      start_features = tf.einsum("lbh,bkl->bkh", output, start_index)
      end_input = tf.tile(output[:, :, None],
                          [1, 1, FLAGS.start_n_top, 1])
      start_features = tf.tile(start_features[None],
                               [seq_len, 1, 1, 1])
      end_input = tf.concat([end_input, start_features], axis=-1)
      end_logits = tf.layers.dense(
          end_input,
          xlnet_config.d_model,
          kernel_initializer=initializer,
          activation=tf.tanh,
          name="dense_0")
      end_logits = tf.contrib.layers.layer_norm(end_logits,
                                                begin_norm_axis=-1)
      end_logits = tf.layers.dense(
          end_logits,
          1,
          kernel_initializer=initializer,
          name="dense_1")
      end_logits = tf.reshape(end_logits, [seq_len, -1, FLAGS.start_n_top])
      end_logits = tf.transpose(end_logits, [1, 2, 0])
      end_logits_masked = end_logits * (
          1 - p_mask[:, None]) - 1e30 * p_mask[:, None]
      end_log_probs = tf.nn.log_softmax(end_logits_masked, -1)
      end_top_log_probs, end_top_index = tf.nn.top_k(
          end_log_probs, k=FLAGS.end_n_top)
      end_top_log_probs = tf.reshape(
          end_top_log_probs,
          [-1, FLAGS.start_n_top * FLAGS.end_n_top])
      end_top_index = tf.reshape(
          end_top_index,
          [-1, FLAGS.start_n_top * FLAGS.end_n_top])

  if is_training:
    return_dict["start_log_probs"] = start_log_probs
    return_dict["end_log_probs"] = end_log_probs
  else:
    return_dict["start_top_log_probs"] = start_top_log_probs
    return_dict["start_top_index"] = start_top_index
    return_dict["end_top_log_probs"] = end_top_log_probs
    return_dict["end_top_index"] = end_top_index

  return return_dict
def get_qa_outputs ( FLAGS , features , is_training ) : inp = tf . transpose ( features [ "input_ids" ] , [ 1 , 0 ] ) seg_id = tf . transpose ( features [ "segment_ids" ] , [ 1 , 0 ] ) inp_mask = tf . transpose ( features [ "input_mask" ] , [ 1 , 0 ] ) seq_len = tf . shape ( inp ) [ 0 ] xlnet_config = xlnet . XLNetConfig ( json_path = FLAGS . model_config_path ) run_config = xlnet . create_run_config ( is_training , True , FLAGS ) xlnet_model = xlnet . XLNetModel ( xlnet_config = xlnet_config , run_config = run_config , input_ids = inp , seg_ids = seg_id , input_mask = inp_mask ) output = xlnet_model . get_sequence_output ( ) initializer = xlnet_model . get_initializer ( ) return_dict = { } # invalid position mask such as query and special symbols (PAD, SEP, CLS) p_mask = features [ "p_mask" ] # logit of the start position with tf . variable_scope ( "start_logits" ) : start_logits = tf . layers . dense ( output , 1 , kernel_initializer = initializer ) start_logits = tf . transpose ( tf . squeeze ( start_logits , - 1 ) , [ 1 , 0 ] ) start_logits_masked = start_logits * ( 1 - p_mask ) - 1e30 * p_mask start_log_probs = tf . nn . log_softmax ( start_logits_masked , - 1 ) # logit of the end position with tf . variable_scope ( "end_logits" ) : if is_training : # during training, compute the end logits based on the # ground truth of the start position start_positions = tf . reshape ( features [ "start_positions" ] , [ - 1 ] ) start_index = tf . one_hot ( start_positions , depth = seq_len , axis = - 1 , dtype = tf . float32 ) start_features = tf . einsum ( "lbh,bl->bh" , output , start_index ) start_features = tf . tile ( start_features [ None ] , [ seq_len , 1 , 1 ] ) end_logits = tf . layers . dense ( tf . concat ( [ output , start_features ] , axis = - 1 ) , xlnet_config . d_model , kernel_initializer = initializer , activation = tf . tanh , name = "dense_0" ) end_logits = tf . contrib . layers . layer_norm ( end_logits , begin_norm_axis = - 1 ) end_logits = tf . layers . dense ( end_logits , 1 , kernel_initializer = initializer , name = "dense_1" ) end_logits = tf . transpose ( tf . squeeze ( end_logits , - 1 ) , [ 1 , 0 ] ) end_logits_masked = end_logits * ( 1 - p_mask ) - 1e30 * p_mask end_log_probs = tf . nn . log_softmax ( end_logits_masked , - 1 ) else : # during inference, compute the end logits based on beam search start_top_log_probs , start_top_index = tf . nn . top_k ( start_log_probs , k = FLAGS . start_n_top ) start_index = tf . one_hot ( start_top_index , depth = seq_len , axis = - 1 , dtype = tf . float32 ) start_features = tf . einsum ( "lbh,bkl->bkh" , output , start_index ) end_input = tf . tile ( output [ : , : , None ] , [ 1 , 1 , FLAGS . start_n_top , 1 ] ) start_features = tf . tile ( start_features [ None ] , [ seq_len , 1 , 1 , 1 ] ) end_input = tf . concat ( [ end_input , start_features ] , axis = - 1 ) end_logits = tf . layers . dense ( end_input , xlnet_config . d_model , kernel_initializer = initializer , activation = tf . tanh , name = "dense_0" ) end_logits = tf . contrib . layers . layer_norm ( end_logits , begin_norm_axis = - 1 ) end_logits = tf . layers . dense ( end_logits , 1 , kernel_initializer = initializer , name = "dense_1" ) end_logits = tf . reshape ( end_logits , [ seq_len , - 1 , FLAGS . start_n_top ] ) end_logits = tf . transpose ( end_logits , [ 1 , 2 , 0 ] ) end_logits_masked = end_logits * ( 1 - p_mask [ : , None ] ) - 1e30 * p_mask [ : , None ] end_log_probs = tf . nn . log_softmax ( end_logits_masked , - 1 ) end_top_log_probs , end_top_index = tf . nn . top_k ( end_log_probs , k = FLAGS . end_n_top ) end_top_log_probs = tf . reshape ( end_top_log_probs , [ - 1 , FLAGS . start_n_top * FLAGS . end_n_top ] ) end_top_index = tf . reshape ( end_top_index , [ - 1 , FLAGS . start_n_top * FLAGS . end_n_top ] ) if is_training : return_dict [ "start_log_probs" ] = start_log_probs return_dict [ "end_log_probs" ] = end_log_probs else : return_dict [ "start_top_log_probs" ] = start_top_log_probs return_dict [ "start_top_index" ] = start_top_index return_dict [ "end_top_log_probs" ] = end_top_log_probs return_dict [ "end_top_index" ] = end_top_index return return_dict