(self, q, k, v, mask)
return output, attention_weights
This is where the layer's logic lives.
This is where the layer's logic lives.
This is where the layer s logic lives .
def call(self, q, k, v, mask):
    """This is where the layer's logic lives."""
    matmul_qk = tf.matmul(q, k, transpose_b=True)  # (..., seq_len_q, seq_len_k)

    # scale matmul_qk
    dk = tf.cast(tf.shape(k)[-1], tf.float32)
    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)

    # add the mask to the scaled tensor.
    if mask is not None:
      scaled_attention_logits += mask * -1e9

    # softmax is normalized on the last axis (seq_len_k) so that the scores
    # add up to 1.
    # (..., seq_len_q, seq_len_k)
    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)

    output = tf.matmul(attention_weights, v)  # (..., seq_len_q, depth_v)

    return output, attention_weights
def call ( self , q , k , v , mask ) : matmul_qk = tf . matmul ( q , k , transpose_b = True ) # (..., seq_len_q, seq_len_k) # scale matmul_qk dk = tf . cast ( tf . shape ( k ) [ - 1 ] , tf . float32 ) scaled_attention_logits = matmul_qk / tf . math . sqrt ( dk ) # add the mask to the scaled tensor. if mask is not None : scaled_attention_logits += mask * - 1e9 # softmax is normalized on the last axis (seq_len_k) so that the scores # add up to 1. # (..., seq_len_q, seq_len_k) attention_weights = tf . nn . softmax ( scaled_attention_logits , axis = - 1 ) output = tf . matmul ( attention_weights , v ) # (..., seq_len_q, depth_v) return output , attention_weights