(qlen, mlen, dtype=tf.float32, same_length=False)
return ret
create causal attention mask.
create causal attention mask.
create causal attention mask .
def _create_mask(qlen, mlen, dtype=tf.float32, same_length=False):
  """create causal attention mask."""
  attn_mask = tf.ones([qlen, qlen], dtype=dtype)
  mask_u = tf.matrix_band_part(attn_mask, 0, -1)
  mask_dia = tf.matrix_band_part(attn_mask, 0, 0)
  attn_mask_pad = tf.zeros([qlen, mlen], dtype=dtype)
  ret = tf.concat([attn_mask_pad, mask_u - mask_dia], 1)
  if same_length:
    mask_l = tf.matrix_band_part(attn_mask, -1, 0)
    ret = tf.concat([ret[:, :qlen] + mask_l - mask_dia, ret[:, qlen:]], 1)

  return ret
def _create_mask ( qlen , mlen , dtype = tf . float32 , same_length = False ) : attn_mask = tf . ones ( [ qlen , qlen ] , dtype = dtype ) mask_u = tf . matrix_band_part ( attn_mask , 0 , - 1 ) mask_dia = tf . matrix_band_part ( attn_mask , 0 , 0 ) attn_mask_pad = tf . zeros ( [ qlen , mlen ] , dtype = dtype ) ret = tf . concat ( [ attn_mask_pad , mask_u - mask_dia ] , 1 ) if same_length : mask_l = tf . matrix_band_part ( attn_mask , - 1 , 0 ) ret = tf . concat ( [ ret [ : , : qlen ] + mask_l - mask_dia , ret [ : , qlen : ] ] , 1 ) return ret