(slen, lengths, causal, padding_mask=None)
return mask, attn_mask
Generate hidden states mask, and optionally an attention mask.
Generate hidden states mask, and optionally an attention mask.
Generate hidden states mask and optionally an attention mask .
def get_masks(slen, lengths, causal, padding_mask=None):
    """
    Generate hidden states mask, and optionally an attention mask.
    """
    bs = lengths.size(0)
    if padding_mask is not None:
        mask = padding_mask
    else:
        assert lengths.max().item() <= slen
        alen = torch.arange(slen, dtype=torch.long, device=lengths.device)
        mask = alen < lengths[:, None]

    # attention mask is the same as mask, or triangular inferior attention (causal)
    if causal:
        attn_mask = alen[None, None, :].repeat(bs, slen, 1) <= alen[None, :, None]
    else:
        attn_mask = mask

    # sanity check
    assert mask.size() == (bs, slen)
    assert causal is False or attn_mask.size() == (bs, slen, slen)

    return mask, attn_mask
def get_masks ( slen , lengths , causal , padding_mask = None ) : bs = lengths . size ( 0 ) if padding_mask is not None : mask = padding_mask else : assert lengths . max ( ) . item ( ) <= slen alen = torch . arange ( slen , dtype = torch . long , device = lengths . device ) mask = alen < lengths [ : , None ] # attention mask is the same as mask, or triangular inferior attention (causal) if causal : attn_mask = alen [ None , None , : ] . repeat ( bs , slen , 1 ) <= alen [ None , : , None ] else : attn_mask = mask # sanity check assert mask . size ( ) == ( bs , slen ) assert causal is False or attn_mask . size ( ) == ( bs , slen , slen ) return mask , attn_mask