Python torch.erf() Examples

The following are 30 code examples of torch.erf(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module torch , or try the search function

Example #1

Source File: normal.py From amortized-variational-filtering with MIT License

6 votes

def cdf(self, value):
        """
        Evaluate the cumulative distribution function at the value.

        Args:
            value (Variable, tensor): the value at which to evaluate the cdf
        """
        n_samples = value.data.shape[1]
        mean = self.mean
        std = self.log_var.mul(0.5).exp_()
        # unsqueeze the parameters along the sample dimension
        if len(mean.size()) == 2:
            mean = mean.unsqueeze(1).repeat(1, n_samples, 1)
            std = std.unsqueeze(1).repeat(1, n_samples, 1)
        elif len(mean.size()) == 4:
            mean = mean.unsqueeze(1).repeat(1, n_samples, 1, 1, 1)
            std = std.unsqueeze(1).repeat(1, n_samples, 1, 1, 1)

        return (1 + torch.erf((value - mean) / (math.sqrt(2) * std).add(1e-5))).mul_(0.5)

Example #2

Source File: modeling.py From KagNet with MIT License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #3

Source File: modeling.py From PPLM with Apache License 2.0

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #4

Source File: activation.py From Double-Branch-Dual-Attention-Mechanism-Network with GNU Affero General Public License v3.0

5 votes

def forward(self, x):
        return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #5

Source File: functional_utils.py From NeMo with Apache License 2.0

5 votes

def gelu(x):
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #6

Source File: modeling.py From RCZoo with MIT License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #7

Source File: activation.py From texar-pytorch with Apache License 2.0

5 votes

def forward(self, x):
        return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #8

Source File: modeling_utils.py From tape with BSD 3-Clause "New" or "Revised" License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different
            (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #9

Source File: modeling.py From SpanABSA with Apache License 2.0

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #10

Source File: modeling.py From lxmert with MIT License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #11

Source File: modeling.py From VLP with Apache License 2.0

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #12

Source File: models.py From pytorchic-bert with Apache License 2.0

5 votes

def gelu(x):
    "Implementation of the gelu activation function by Hugging Face"
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #13

Source File: modeling_bert.py From Bert-Multi-Label-Text-Classification with MIT License

5 votes

def gelu(x):
    """ Original Implementation of the gelu activation function in Google Bert repo when initially created.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #14

Source File: math.py From uncertainty_estimation_deep_learning with MIT License

5 votes

def normcdf(value, mu=0.0, stddev=1.0):
    sinv = (1.0 / stddev) if isinstance(stddev, Number) else stddev.reciprocal()
    return 0.5 * (1.0 + torch.erf((value - mu) * sinv / np.sqrt(2.0)))

Example #15

Source File: bert.py From attention-cnn with Apache License 2.0

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #16

Source File: modeling.py From MTMSN with Apache License 2.0

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #17

Source File: modeling_albert.py From Bert-Multi-Label-Text-Classification with MIT License

5 votes

def gelu(x):
    """ Original Implementation of the gelu activation function in Google Bert repo when initially created.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #18

Source File: transformer.py From dl4mt-seqgen with BSD 3-Clause "New" or "Revised" License

5 votes

def gelu(x):
    """
    GELU activation
    https://arxiv.org/abs/1606.08415
    https://github.com/huggingface/pytorch-openai-transformer-lm/blob/master/model_pytorch.py#L14
    https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/modeling.py
    """
    # return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    return 0.5 * x * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #19

Source File: bert_maxout_clf.py From semanticRetrievalMRS with MIT License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #20

Source File: bert_multilayer_output.py From semanticRetrievalMRS with MIT License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #21

Source File: utils.py From gtos with MIT License

5 votes

def gelu(x: torch.Tensor) -> torch.Tensor:
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #22

Source File: utils.py From gtos with MIT License

5 votes

def gelu(x: torch.Tensor) -> torch.Tensor:
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #23

Source File: activations.py From exbert with Apache License 2.0

5 votes

def _gelu_python(x):
    """ Original Implementation of the gelu activation function in Google Bert repo when initially created.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        This is now written in C in torch.nn.functional
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #24

Source File: Bert_modeling.py From TriB-QA with MIT License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
        Also see https://arxiv.org/abs/1606.08415
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #25

Source File: loss.py From oft with MIT License

5 votes

def log_ap_loss(logvar, sqr_dists, num_thresh=10):


    print('dists', float(sqr_dists.min()), float(sqr_dists.max()))
    print('logvar', float(logvar.min()), float(logvar.max()))

    def hook(grad):
        print('grad', float(grad.min()), float(grad.max()), float(grad.sum()))
    logvar.register_hook(hook)

    variance = torch.exp(logvar).view(-1, 1)
    stdev = torch.sqrt(variance)
    print('stdev', float(stdev.min()), float(stdev.max()))

    max_dist = math.sqrt(float(sqr_dists.max()))
    minvar, maxvar = float(stdev.min()), float(stdev.max())
    thresholds = torch.logspace(
        math.log10(1 / maxvar), math.log10(max_dist / minvar), num_thresh).type_as(stdev)
    
    print('maxdist: {:.2e} minvar: {:.2e} maxvar: {:.2e}'.format(max_dist, minvar, maxvar))
    print('thresholds {:.2e} - {:.2e}'.format(thresholds.min(), thresholds.max()))

    k_sigma = stdev * thresholds
    k_sigma_sqr = variance * thresholds ** 2
    mask = (sqr_dists.view(-1, 1) < k_sigma_sqr).float()

    erf = torch.erf(k_sigma)
    masked_erf = erf * mask
    masked_exp = stdev * torch.exp(-k_sigma_sqr) * mask

    loss = masked_exp.sum(0) * masked_erf.sum(0) / erf.sum(0)
    loss = (loss[0] + loss[-1]) / 2. + loss[1:-1].sum()
    return -torch.log(loss * CONST / len(variance))

Example #26

Source File: gelu.py From neural_sp with Apache License 2.0

5 votes

def gelu(x):
    if hasattr(torch.nn.functional, 'gelu'):
        return torch.nn.functional.gelu(x.float()).type_as(x)
    else:
        return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #27

Source File: utils.py From Guyu with MIT License

5 votes

def gelu(x):
    cdf = 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))
    return cdf*x

Example #28

Source File: modeling.py From unilm with MIT License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #29

Source File: modeling_decoding.py From unilm with MIT License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

Example #30

Source File: modeling.py From SDNet with MIT License

5 votes

def gelu(x):
    """Implementation of the gelu activation function.
        For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):
        0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3))))
    """
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))