(q, k, v, mask)
return output, attention_weights
The implementation of scaled attention.
    Args:
      v: (batch_size, seq_len_v, hidden_size)
      k: (batch_size, seq_len_k, hidden_size)
      q: (batch_size, seq_len_q, hidden_size)
      mask: (batch_size, seq_len_q, seq_len_k)

    Returns:
      output: (batch_size, seq_len_q, hidden_size)
      attention_weights: (batch_size, num_heads, seq_len_q, seq_len_k)
The implementation of scaled attention.
    Args:
      v: (batch_size, seq_len_v, hidden_size)
      k: (batch_size, seq_len_k, hidden_size)
      q: (batch_size, seq_len_q, hidden_size)
      mask: (batch_size, seq_len_q, seq_len_k)
The implementation of scaled attention . Args : v : ( batch_size seq_len_v hidden_size ) k : ( batch_size seq_len_k hidden_size ) q : ( batch_size seq_len_q hidden_size ) mask : ( batch_size seq_len_q seq_len_k )
def scaled_dot_product_attention(q, k, v, mask):
    """
    The implementation of scaled attention.
    Args:
      v: (batch_size, seq_len_v, hidden_size)
      k: (batch_size, seq_len_k, hidden_size)
      q: (batch_size, seq_len_q, hidden_size)
      mask: (batch_size, seq_len_q, seq_len_k)

    Returns:
      output: (batch_size, seq_len_q, hidden_size)
      attention_weights: (batch_size, num_heads, seq_len_q, seq_len_k)
    """

    matmul_qk = tf.matmul(
        q, k, transpose_b=True)  # (batch_size, seq_len_q, seq_len_k)

    # Scaled
    dk = tf.cast(tf.shape(k)[-1], tf.float32)
    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)

    # Masked
    if mask is not None:
      scaled_attention_logits += (mask * -1e9)

    # Normalized
    attention_weights = tf.nn.softmax(
        scaled_attention_logits, axis=-1)  # (batch_size, seq_len_q, seq_len_k)

    # Weighted sum
    output = tf.matmul(attention_weights, v)  # (batch_size, seq_len_q, depth_v)

    return output, attention_weights
def scaled_dot_product_attention ( q , k , v , mask ) : matmul_qk = tf . matmul ( q , k , transpose_b = True ) # (batch_size, seq_len_q, seq_len_k) # Scaled dk = tf . cast ( tf . shape ( k ) [ - 1 ] , tf . float32 ) scaled_attention_logits = matmul_qk / tf . math . sqrt ( dk ) # Masked if mask is not None : scaled_attention_logits += ( mask * - 1e9 ) # Normalized attention_weights = tf . nn . softmax ( scaled_attention_logits , axis = - 1 ) # (batch_size, seq_len_q, seq_len_k) # Weighted sum output = tf . matmul ( attention_weights , v ) # (batch_size, seq_len_q, depth_v) return output , attention_weights