(bert_model, labels)
return embedding
Generate bert's embedding from fine-tuned model.
Generate bert's embedding from fine-tuned model.
Generate bert s embedding from fine - tuned model .
def generate_embedding(bert_model, labels):
    """Generate bert's embedding from fine-tuned model."""
    batch_size, time = labels.shape

    cls_ids = torch.full(
        (batch_size, 1), bert_model.bert_text_encoder.cls_idx, dtype=labels.dtype, device=labels.device)
    bert_labels = torch.cat([cls_ids, labels], 1)
    # replace eos with sep
    eos_idx = bert_model.bert_text_encoder.eos_idx
    sep_idx = bert_model.bert_text_encoder.sep_idx
    bert_labels[bert_labels == eos_idx] = sep_idx

    embedding, _ = bert_model.bert(bert_labels, output_all_encoded_layers=True)
    # sum over all layers embedding
    embedding = torch.stack(embedding).sum(0)
    # get rid of cls
    embedding = embedding[:, 1:]

    assert labels.shape == embedding.shape[:-1]

    return embedding
def generate_embedding ( bert_model , labels ) : batch_size , time = labels . shape cls_ids = torch . full ( ( batch_size , 1 ) , bert_model . bert_text_encoder . cls_idx , dtype = labels . dtype , device = labels . device ) bert_labels = torch . cat ( [ cls_ids , labels ] , 1 ) # replace eos with sep eos_idx = bert_model . bert_text_encoder . eos_idx sep_idx = bert_model . bert_text_encoder . sep_idx bert_labels [ bert_labels == eos_idx ] = sep_idx embedding , _ = bert_model . bert ( bert_labels , output_all_encoded_layers = True ) # sum over all layers embedding embedding = torch . stack ( embedding ) . sum ( 0 ) # get rid of cls embedding = embedding [ : , 1 : ] assert labels . shape == embedding . shape [ : - 1 ] return embedding