```python
import numpy as np
import matplotlib.pyplot as plt
from sympy import *
```


```python
# create function
x = Symbol('x')
f = x**2 + 8*sin(x)

# derivative
df = f.diff(x)

# lambdify both
f = lambdify(x, f)
df = lambdify(x, df)

# evaluate
pts = np.linspace(-5, 8)
plt.plot(pts, f(pts), label='f(x)')
plt.plot(pts, df(pts), label='df/dx')
plt.legend()
plt.show()
```


```python
max_iter = 100
momentum = 0.8
moment = [0]
with_momentum = [False, True]

strategy_steps = []

for use_momentum in with_momentum:
    lr = 0.05
    X0 = 10
    cur_iter = 0
    steps = [X0]
    while True:
        # sum last 10 momentum
        m = sum(moment[-10:])
        # get closer to a local min
        grad = lr * df(X0)
        if use_momentum:
            X1 = X0 + m - grad
        else:
            X1 = X0 - grad
        # save step
        steps.append(X1)
        # save grad
        moment.append(-grad)
        # previous moment are : 0.9*(grad N), 0.9*0.9*(grad N-1), etc.
        moment = [momentum*x for x in moment]
        # calculate step size
        step_size = X1 - X0
        # save new step
        X0 = X1
        if cur_iter > max_iter or abs(step_size) < 0.00001:
            print("Convergence done")
            break
        elif cur_iter%40 == 0:
            lr = lr*0.9
            print("Iteration {} has x value {}".format(cur_iter, X1))
        cur_iter = cur_iter + 1
    strategy_steps.append(steps)
    
print("The local minimum occurs at", X1)
```

    Iteration 0 has x value 9.33562861163058
    Iteration 40 has x value 3.5953497384665702
    Convergence done
    Iteration 0 has x value 9.335540104209164
    Iteration 40 has x value -1.2902349357717582
    Iteration 80 has x value -1.2513700135960928
    Convergence done
    The local minimum occurs at -1.2523427452482645



```python
fig = plt.figure(num=None, figsize=(8, 10), dpi=80, facecolor='w', edgecolor='k')
ax1 = fig.add_subplot(2, 1, 1)
steps = strategy_steps[0]
pts = np.linspace(-5, 8)
# change color to see GD progress
color = [item/255 for item in list(range(len(steps)))]
ax1.plot(pts, f(pts), label='f(x)')
ax1.scatter(steps, [f(x) for x in steps], label='GD steps', c=color)
ax1.scatter(steps[0], f(steps[0]), label='Start GD', c='red', s=50)
ax1.scatter(steps[-1], f(steps[-1]), label='End GD', c='yellow', s=50)
plt.title('Gradient Descent without momentum stays in local minimum')
ax1.legend()

ax2 = fig.add_subplot(2, 1, 2)
steps = strategy_steps[1]
pts = np.linspace(-5, 8)
# change color to see GD progress
color = [item/255 for item in list(range(len(steps)))]
ax2.plot(pts, f(pts), label='f(x)')
ax2.scatter(steps, [f(x) for x in steps], label='GD steps', c=color)
ax2.scatter(steps[0], f(steps[0]), label='Start GD', c='red', s=50)
ax2.scatter(steps[-1], f(steps[-1]), label='End GD', c='yellow', s=50)
plt.title('Gradient descent with momentum can avoid local minimum')
ax2.legend()

plt.show()
fig.savefig('../plots/2D_GD.png')
```

# Animation


```python
from tqdm import tqdm
from PIL import Image
def fig2img(fig):
    """Convert a Matplotlib figure to a PIL Image and return it"""
    import io
    buf = io.BytesIO()
    fig.savefig(buf)
    buf.seek(0)
    img = Image.open(buf)
    return img
```


```python
images = []
fig = plt.figure(num=None, figsize=(8, 10), dpi=80, facecolor='w', edgecolor='k')
ax1 = fig.add_subplot(2, 1, 1)
ax1.plot(pts, f(pts), c='grey')
ax2 = fig.add_subplot(2, 1, 2)
ax2.plot(pts, f(pts), c='grey')
steps1 = strategy_steps[0]
x1, y1, intensity1 = [], [], []

steps2 = strategy_steps[1]
x2, y2, intensity2 = [], [], []
for i, step in enumerate(tqdm(steps1)):
    pts = np.linspace(-5, 8)
    x1.append(step)
    y1.append(f(step))
    intensity1.append(i)
    ax1.scatter(x1, y1, cmap = 'viridis', c=intensity1)
    ax1.set_title('Gradient Descent without momentum stays in local minimum')
    images.append(fig2img(fig))

    step = steps2[i]
    pts = np.linspace(-5, 8)
    x2.append(step)
    y2.append(f(step))
    intensity2.append(i)
    ax2.scatter(x2, y2, cmap = 'viridis', c=intensity2)
    ax2.set_title('Gradient descent with momentum can avoid local minimum')
    images.append(fig2img(fig))
```


```python
images[0].save(fp='../plots/2D_GD.gif', format='GIF', append_images=images,
               save_all=True, duration=100, loop=0)
```


```python

```

48ad012b80eecc1796ecead6ba022cc60bdc0e8f
ipynb
Jupyter Notebook
differential_approach/gradient_descent_momentum.ipynb
Apiquet/gradient_descent
ce4431c88e84e55ccd633bfc10e3a82ac3c933c0
MIT
differential_approach/gradient_descent_momentum.ipynb
Apiquet/gradient_descent
ce4431c88e84e55ccd633bfc10e3a82ac3c933c0
MIT
differential_approach/gradient_descent_momentum.ipynb
Apiquet/gradient_descent
ce4431c88e84e55ccd633bfc10e3a82ac3c933c0
MIT
Qwen/Qwen-72B
1. YES
2. YES