(dec_inp, target, mems, n_token, n_layer, d_model, d_embed,
                n_head, d_head, d_inner, dropout, dropatt,
                initializer, is_training, proj_initializer=None,
                mem_len=None, cutoffs=[], div_val=1, tie_projs=[],
                same_length=False, clamp_len=-1,
                input_perms=None, target_perms=None, head_target=None,
                untie_r=False, proj_same_dim=True,
                scope='transformer')
cutoffs: a list of python int. Cutoffs for adaptive softmax.
    tie_projs: a list of python bools. Whether to tie the projections.
    perms: a list of tensors. Each tensor should of size [len, bsz, bin_size].
        Only used in the adaptive setting.
cutoffs: a list of python int. Cutoffs for adaptive softmax.
    tie_projs: a list of python bools. Whether to tie the projections.
    perms: a list of tensors. Each tensor should of size [len, bsz, bin_size].
        Only used in the adaptive setting.
cutoffs : a list of python int . Cutoffs for adaptive softmax . tie_projs : a list of python bools . Whether to tie the projections . perms : a list of tensors . Each tensor should of size [ len bsz bin_size ] . Only used in the adaptive setting .
def transformer(dec_inp, target, mems, n_token, n_layer, d_model, d_embed,
                n_head, d_head, d_inner, dropout, dropatt,
                initializer, is_training, proj_initializer=None,
                mem_len=None, cutoffs=[], div_val=1, tie_projs=[],
                same_length=False, clamp_len=-1,
                input_perms=None, target_perms=None, head_target=None,
                untie_r=False, proj_same_dim=True,
                scope='transformer'):
    """
    cutoffs: a list of python int. Cutoffs for adaptive softmax.
    tie_projs: a list of python bools. Whether to tie the projections.
    perms: a list of tensors. Each tensor should of size [len, bsz, bin_size].
        Only used in the adaptive setting.
    """
    new_mems = []
    with tf.compat.v1.variable_scope(scope):
        if untie_r:
            r_w_bias = tf.compat.v1.get_variable('r_w_bias', [n_layer, n_head, d_head], initializer=initializer)
            r_r_bias = tf.compat.v1.get_variable('r_r_bias', [n_layer, n_head, d_head], initializer=initializer)
        else:
            r_w_bias = tf.compat.v1.get_variable('r_w_bias', [n_head, d_head], initializer=initializer)
            r_r_bias = tf.compat.v1.get_variable('r_r_bias', [n_head, d_head], initializer=initializer)

        qlen = tf.shape(dec_inp)[0]
        mlen = tf.shape(mems[0])[0] if mems is not None else 0
        klen = qlen + mlen

        if proj_initializer is None:
            proj_initializer = initializer

        embeddings, shared_params = normal_embedding_lookup(
            x=dec_inp,
            n_token=n_token,
            d_embed=d_embed,
            d_proj=d_model,
            initializer=initializer,
            proj_initializer=proj_initializer)
        
        attn_mask = _create_mask(qlen, mlen, same_length)
        
        pos_seq = tf.range(klen - 1, -1, -1.0)
        if clamp_len > 0:
            pos_seq = tf.minimum(pos_seq, clamp_len)
        inv_freq = 1 / (10000 ** (tf.range(0, d_model, 2.0) / d_model))
        pos_emb = positional_embedding(pos_seq, inv_freq)

        output = tf.keras.layers.Dropout(rate=dropout)(embeddings, training=is_training)
        pos_emb = tf.keras.layers.Dropout(rate=dropout)(pos_emb, training=is_training)

        if mems is None:
            mems = [None] * n_layer

        for i in range(n_layer):
            # cache new mems
            new_mems.append(_cache_mem(output, mems[i], mem_len))

            with tf.compat.v1.variable_scope('layer_{}'.format(i)):
                output = rel_multihead_attn(
                    w=output,
                    r=pos_emb,
                    r_w_bias=r_w_bias if not untie_r else r_w_bias[i],
                    r_r_bias=r_r_bias if not untie_r else r_r_bias[i],
                    attn_mask=attn_mask,
                    mems=mems[i],
                    d_model=d_model,
                    n_head=n_head,
                    d_head=d_head,
                    dropout=dropout,
                    dropatt=dropatt,
                    is_training=is_training,
                    kernel_initializer=initializer)

                output = positionwise_FF(
                    inp=output,
                    d_model=d_model,
                    d_inner=d_inner,
                    dropout=dropout,
                    kernel_initializer=initializer,
                    is_training=is_training)

        output = tf.keras.layers.Dropout(dropout)(output, training=is_training)

        loss, logits = normal_softmax(
            hidden=output,
            target=target,
            n_token=n_token,
            params=shared_params)

        return loss, logits, new_mems
def transformer ( dec_inp , target , mems , n_token , n_layer , d_model , d_embed , n_head , d_head , d_inner , dropout , dropatt , initializer , is_training , proj_initializer = None , mem_len = None , cutoffs = [ ] , div_val = 1 , tie_projs = [ ] , same_length = False , clamp_len = - 1 , input_perms = None , target_perms = None , head_target = None , untie_r = False , proj_same_dim = True , scope = 'transformer' ) : new_mems = [ ] with tf . compat . v1 . variable_scope ( scope ) : if untie_r : r_w_bias = tf . compat . v1 . get_variable ( 'r_w_bias' , [ n_layer , n_head , d_head ] , initializer = initializer ) r_r_bias = tf . compat . v1 . get_variable ( 'r_r_bias' , [ n_layer , n_head , d_head ] , initializer = initializer ) else : r_w_bias = tf . compat . v1 . get_variable ( 'r_w_bias' , [ n_head , d_head ] , initializer = initializer ) r_r_bias = tf . compat . v1 . get_variable ( 'r_r_bias' , [ n_head , d_head ] , initializer = initializer ) qlen = tf . shape ( dec_inp ) [ 0 ] mlen = tf . shape ( mems [ 0 ] ) [ 0 ] if mems is not None else 0 klen = qlen + mlen if proj_initializer is None : proj_initializer = initializer embeddings , shared_params = normal_embedding_lookup ( x = dec_inp , n_token = n_token , d_embed = d_embed , d_proj = d_model , initializer = initializer , proj_initializer = proj_initializer ) attn_mask = _create_mask ( qlen , mlen , same_length ) pos_seq = tf . range ( klen - 1 , - 1 , - 1.0 ) if clamp_len > 0 : pos_seq = tf . minimum ( pos_seq , clamp_len ) inv_freq = 1 / ( 10000 ** ( tf . range ( 0 , d_model , 2.0 ) / d_model ) ) pos_emb = positional_embedding ( pos_seq , inv_freq ) output = tf . keras . layers . Dropout ( rate = dropout ) ( embeddings , training = is_training ) pos_emb = tf . keras . layers . Dropout ( rate = dropout ) ( pos_emb , training = is_training ) if mems is None : mems = [ None ] * n_layer for i in range ( n_layer ) : # cache new mems new_mems . append ( _cache_mem ( output , mems [ i ] , mem_len ) ) with tf . compat . v1 . variable_scope ( 'layer_{}' . format ( i ) ) : output = rel_multihead_attn ( w = output , r = pos_emb , r_w_bias = r_w_bias if not untie_r else r_w_bias [ i ] , r_r_bias = r_r_bias if not untie_r else r_r_bias [ i ] , attn_mask = attn_mask , mems = mems [ i ] , d_model = d_model , n_head = n_head , d_head = d_head , dropout = dropout , dropatt = dropatt , is_training = is_training , kernel_initializer = initializer ) output = positionwise_FF ( inp = output , d_model = d_model , d_inner = d_inner , dropout = dropout , kernel_initializer = initializer , is_training = is_training ) output = tf . keras . layers . Dropout ( dropout ) ( output , training = is_training ) loss , logits = normal_softmax ( hidden = output , target = target , n_token = n_token , params = shared_params ) return loss , logits , new_mems