# Mish Derivatves


```python
import torch
from torch.nn import functional as F
```


```python
inp = torch.randn(100) + (torch.arange(0, 1000, 10, dtype=torch.float)-500.)
inp
```




    tensor([-500.3069, -490.6361, -480.3858, -471.2755, -459.0872, -451.1570,
            -440.2400, -429.6230, -419.9467, -408.3055, -402.3395, -389.1660,
            -380.1614, -369.7649, -359.7261, -348.8759, -338.7170, -329.2680,
            -319.7470, -309.6079, -301.3083, -290.9236, -279.3832, -267.8622,
            -259.3479, -249.7400, -240.8742, -229.2343, -219.3999, -210.0166,
            -199.8259, -191.5603, -178.9595, -171.4488, -160.3362, -150.1327,
            -139.2230, -130.8046, -121.8909, -108.4913, -100.5724,  -88.9087,
             -79.9365,  -70.3478,  -60.1005,  -49.9595,  -37.6322,  -29.9353,
             -18.9407,  -11.9213,   -2.5633,   10.6869,   18.9005,   29.4622,
              41.7188,   49.6080,   59.3583,   71.3071,   80.2604,   91.4908,
             100.2913,  108.7626,  118.8391,  129.8859,  139.5593,  150.6612,
             161.5152,  170.5409,  179.0472,  187.5896,  199.0938,  210.3955,
             221.2551,  229.1151,  240.5497,  250.7286,  260.3474,  268.6524,
             280.6704,  291.0199,  302.0525,  309.1079,  320.3692,  330.5589,
             340.5503,  350.1638,  359.5840,  369.0214,  379.5835,  390.5975,
             398.7851,  408.7201,  420.3418,  430.0718,  440.4431,  449.6514,
             459.1114,  468.1187,  480.6921,  490.0807])




```python
import sympy
from sympy import Symbol, Function, Expr, diff, simplify, exp, log, tanh
x = Symbol('x')
f =  Function('f')
```

## Overall Derivative


```python
diff(x*tanh(log(exp(x)+1)))
```




$\displaystyle \frac{x \left(1 - \tanh^{2}{\left(\log{\left(e^{x} + 1 \right)} \right)}\right) e^{x}}{e^{x} + 1} + \tanh{\left(\log{\left(e^{x} + 1 \right)} \right)}$




```python
simplify(diff(x*tanh(log(exp(x)+1))))
```




$\displaystyle - \frac{x \left(\tanh^{2}{\left(\log{\left(e^{x} + 1 \right)} \right)} - 1\right) e^{x} - \left(e^{x} + 1\right) \tanh{\left(\log{\left(e^{x} + 1 \right)} \right)}}{e^{x} + 1}$



## Softplus

$ \Large \frac{\partial}{\partial x} Softplus(x) = 1 - \frac{1}{e^{x} + 1} $

Or, from PyTorch:

$ \Large \frac{\partial}{\partial x} Softplus(x) = 1 - e^{-Y} $

Where $Y$ is saved output


```python
class SoftPlusTest(torch.autograd.Function):
    @staticmethod
    def forward(ctx, inp, threshold=20):
        y = torch.where(inp < threshold, torch.log1p(torch.exp(inp)), inp)
        ctx.save_for_backward(y)
        return y
    
    @staticmethod
    def backward(ctx, grad_out):
        y, = ctx.saved_tensors
        res = 1 - (-y).exp_()
        return grad_out * res

```


```python
torch.allclose(F.softplus(inp), SoftPlusTest.apply(inp))
```




    True




```python
torch.autograd.gradcheck(SoftPlusTest.apply, inp.to(torch.float64).requires_grad_())
```




    True



## $tanh(Softplus(x))$


```python
diff(tanh(f(x)))
```




$\displaystyle \left(1 - \tanh^{2}{\left(f{\left(x \right)} \right)}\right) \frac{d}{d x} f{\left(x \right)}$




```python
class TanhSPTest(torch.autograd.Function):
    @staticmethod
    def forward(ctx, inp, threshold=20):
        ctx.save_for_backward(inp)
        sp = torch.where(inp < threshold, torch.log1p(torch.exp(inp)), inp)
        y = torch.tanh(sp)
        return y
    
    @staticmethod
    def backward(ctx, grad_out, threshold=20):
        inp, = ctx.saved_tensors
        sp = torch.where(inp < threshold, torch.log1p(torch.exp(inp)), inp)
        grad_sp = 1 - torch.exp(-sp)
        tanhsp = torch.tanh(sp)
        grad = (1 - tanhsp*tanhsp) * grad_sp
        return grad_out * grad

```


```python
torch.allclose(TanhSPTest.apply(inp), torch.tanh(F.softplus(inp)))
```




    True




```python
torch.autograd.gradcheck(TanhSPTest.apply, inp.to(torch.float64).requires_grad_())
```




    True



## Mish


```python
diff(x * f(x))
```




$\displaystyle x \frac{d}{d x} f{\left(x \right)} + f{\left(x \right)}$




```python
diff(x*tanh(f(x)))
```




$\displaystyle x \left(1 - \tanh^{2}{\left(f{\left(x \right)} \right)}\right) \frac{d}{d x} f{\left(x \right)} + \tanh{\left(f{\left(x \right)} \right)}$




```python
simplify(diff(x*tanh(f(x))))
```




$\displaystyle \frac{x \frac{d}{d x} f{\left(x \right)}}{\cosh^{2}{\left(f{\left(x \right)} \right)}} + \tanh{\left(f{\left(x \right)} \right)}$




```python
diff(tanh(f(x)))
```




$\displaystyle \left(1 - \tanh^{2}{\left(f{\left(x \right)} \right)}\right) \frac{d}{d x} f{\left(x \right)}$




```python
class MishTest(torch.autograd.Function):
    @staticmethod
    def forward(ctx, inp, threshold=20):
        ctx.save_for_backward(inp)
        sp = torch.where(inp < threshold, torch.log1p(torch.exp(inp)), inp)
        tsp = torch.tanh(sp)
        y = inp.mul(tsp)
        return y
    
    @staticmethod
    def backward(ctx, grad_out, threshold=20):
        inp, = ctx.saved_tensors
        sp = torch.where(inp < threshold, torch.log1p(torch.exp(inp)), inp)
        grad_sp = 1 - torch.exp(-sp)
        tsp = torch.tanh(sp)
        grad_tsp = (1 - tsp*tsp) * grad_sp
        grad = inp * grad_tsp + tsp
        return grad_out * grad

```


```python
torch.allclose(MishTest.apply(inp), inp.mul(torch.tanh(F.softplus(inp))))
```




    True




```python
torch.autograd.gradcheck(TanhSPTest.apply, inp.to(torch.float64).requires_grad_())
```




    True



53d868ef3912cc35ae9fdbb2c8085b7ab875169f
ipynb
Jupyter Notebook
extra/Derivatives.ipynb
hiyyg/mish-cuda
b389b9f84433d8b9b4129d3e879ba746d248d8f2
MIT
2019-09-25T17:43:54.000Z
2022-03-09T08:17:44.000Z
extra/Derivatives.ipynb
hiyyg/mish-cuda
b389b9f84433d8b9b4129d3e879ba746d248d8f2
MIT
2019-11-18T22:20:02.000Z
2022-02-16T03:04:30.000Z
extra/Derivatives.ipynb
hiyyg/mish-cuda
b389b9f84433d8b9b4129d3e879ba746d248d8f2
MIT
2019-10-10T03:52:05.000Z
2022-03-24T07:14:01.000Z
Qwen/Qwen-72B
1. YES
2. YES