(x)
return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
        Also see https://arxiv.org/abs/1606.08415
Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
        Also see https://arxiv.org/abs/1606.08415
Implementation of the gelu activation function currently in Google Bert repo ( identical to OpenAI GPT ) . Also see https : // arxiv . org / abs / 1606 . 08415
def gelu_new(x):
    """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT).
        Also see https://arxiv.org/abs/1606.08415
    """
    return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
def gelu_new ( x ) : return 0.5 * x * ( 1 + torch . tanh ( math . sqrt ( 2 / math . pi ) * ( x + 0.044715 * torch . pow ( x , 3 ) ) ) )