## <span style="color:blue"> Linear Regression using Gradient Descent

Video Explanation :  https://youtu.be/CzW0w7guvFo


```python
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
```


```python
X = [3,4,6,12,9,15,10,1,8,13] ## Experience
y = [16,29,43,65,51,89,57,9,53,68]  ## Salary
```


```python
#X = [3,4,6]
#y = [16,29,43]
```


```python
plt.scatter(X,y)
plt.xlabel('Experience')
plt.ylabel('Salary')
plt.grid(alpha=0.25)
plt.show()
```


```python
# Lets lienar regession line is y = mx + c
# lets 
# y = mx +c
c=0
# # y_pred = mx
# calcaulating the y_out we will mutiply the original X values by the slope to produce y_out values for each X
def multiply_matrix(x,m,c):
    y_out = []
    for i in range(len(x)):
        y_out.append(x[i]*m + c)
    return y_out
```


```python
# Lets 
c = 0
m = 1
y_m1=multiply_matrix(X,m,c)
y_m1
```




    [3, 4, 6, 12, 9, 15, 10, 1, 8, 13]




```python
# line at m =1
plt.scatter(X, y,label='Scatter Plot') 
plt.plot([min(X), max(X)], [min(y_m1), max(y_m1)], color='red',label='Regression best fit Line at m=1')
plt.xlabel('Experience')
plt.ylabel('Salary')
plt.grid(alpha=0.25)
plt.legend()
plt.show()
```


```python
# calculate cost function 1/n*sum(y-y_out)^2 by looping each sample
# calculate mse 1/n*sum(y-y_out)^2 by looping each sample
def mse(actual, predicted):
    sum_error = 0.0
    # loop over all values
    for i in range(len(actual)):
        # the error is the sum of (actual - prediction)^2
        SSE =  (actual[i] - predicted[i])**2
        sum_error =  sum_error + SSE
    mean_error = sum_error / float(len(actual))
    return (mean_error)
```


```python
mse_m1 = mse(y,y_m1)
mse_m1
```




    1953.5




```python
# calculate mse for different m values ,lets take m = range(0,9)
mse_values = []  # store the MSE for each m where m = range(0,9)
m = range(0,9)
c = 0
for i in m: 
    y_out_values = multiply_matrix(X, m[i],c)
    mse_values.append(mse(y, y_out_values))
    print("mse for m :",m[i], "and c :",c ," is ", mse(y, y_out_values))
    plt.figure(figsize=(3,2))
    plt.scatter(X, y,label='Scatter Plot') 
    plt.plot([min(X), max(X)], [min(y_out_values), max(y_out_values)], color='red',label='regression line')
    plt.xlabel('Experience')
    plt.ylabel('Salary')
    plt.grid(alpha=0.25)
    #plt.legend()
    plt.show()
```


```python
plt.plot(m,mse_values,label = "cost function line")
plt.xlabel('slope')
plt.ylabel('Cost Function')
plt.grid(alpha=0.25)
plt.legend()
```


```python
np.min(mse_values)  # at this MSE we get the best bit line when m = 5 and c =0
# but we have to iterate the both m & c to get the best bit line taht we can get with the help 
# of gradient descent(taking derivative of MSE wrt m and c)
```




    28.0



<b>Cost Function</b>

\begin{equation}
Linear Equation ==> y = mx + c
\end{equation}

\begin{equation}
J(x) = 1/n \sum_{i=1}^{n} (y^{(i)} - y^{(i)}predicted)^2 
\end{equation}

\begin{equation}
y^{(i)}predicted = (mx^{(i)} + c)
\end{equation}

\begin{equation}
J(x) = 1/n \sum_{i=1}^{n} (y^{(i)} - (mx^{(i)} + c))^2 
\end{equation}

<b>Gradient</b>

**Derivative of cost function with respect to   m   ==>  slope**
\begin{equation}
\frac{\partial J(x^{(i)})}{\partial m} = -2/n\sum_{i=1}^{n}(x^{(i)}).(y^{(i)} - (mx^{(i)} + c))
\end{equation}

**Derivative of cost function with respect to   c   ==>  intercept**
\begin{equation}
\frac{\partial J(x^{(i)})}{\partial c} = -2/n\sum_{i=1}^{n}(y^{(i)} - (mx^{(i)} + c))
\end{equation}

**Why do we take derivative?**

To get the direction of where is global minimum. If we are on left side of minimum value of parabola then the derivate is negative and negative sign before derivate will make the new value of X go right and thus more closer to minimum value. Vice versa for right side.


```python
import numpy as np
def Simple_LinearRegression_Gradient_Descent(x,y,epochs,learning_rate,n):
    m_curr = c_curr = 0
    for i in range(epochs):
        y_pred = m_curr * x + c_curr
        ## MSE equation == cost_function
        cost = (1/n) * sum([val**2 for val in (y-y_pred)])
        md = -(2/n)*sum(x*(y-y_pred))
        cd = -(2/n)*sum(y-y_pred)
        m_curr = m_curr - learning_rate * md  # step_size = learning_rate * md 
        c_curr = c_curr - learning_rate * cd  # # step_size = learning_rate * cd 
        print('>epoch=%d, m=%.3f,c=%.3f, md=%.3f,cd=%.3f,step_size_m=%.3f,step_size_c=%.3f,cost=%.3f' % 
              (i,m_curr,c_curr, md,cd,md*learning_rate,cd*learning_rate,(cost)))
```


```python
x = np.array([3,4,6,12,9,15,10,1,8,13])
y = np.array([16,29,43,65,51,89,57,9,53,68])
epochs = 30
n = len(x)
learning_rate = 0.001
Simple_LinearRegression_Gradient_Descent(x,y,epochs,learning_rate,n)
```

    >epoch=0, m=0.977,c=0.096, md=-976.600,cd=-96.000,step_size_m=-0.977,step_size_c=-0.096,cost=2845.600
    >epoch=1, m=1.787,c=0.176, md=-809.999,cd=-79.987,step_size_m=-0.810,step_size_c=-0.080,cost=1964.756
    >epoch=2, m=2.458,c=0.243, md=-671.814,cd=-66.705,step_size_m=-0.672,step_size_c=-0.067,cost=1358.756
    >epoch=3, m=3.016,c=0.298, md=-557.197,cd=-55.688,step_size_m=-0.557,step_size_c=-0.056,cost=941.840
    >epoch=4, m=3.478,c=0.345, md=-462.128,cd=-46.550,step_size_m=-0.462,step_size_c=-0.047,cost=655.012
    >epoch=5, m=3.861,c=0.384, md=-383.274,cd=-38.971,step_size_m=-0.383,step_size_c=-0.039,cost=457.679
    >epoch=6, m=4.179,c=0.417, md=-317.870,cd=-32.684,step_size_m=-0.318,step_size_c=-0.033,cost=321.917
    >epoch=7, m=4.443,c=0.444, md=-263.620,cd=-27.469,step_size_m=-0.264,step_size_c=-0.027,cost=228.515
    >epoch=8, m=4.661,c=0.467, md=-218.623,cd=-23.143,step_size_m=-0.219,step_size_c=-0.023,cost=164.256
    >epoch=9, m=4.842,c=0.487, md=-181.301,cd=-19.555,step_size_m=-0.181,step_size_c=-0.020,cost=120.045
    >epoch=10, m=4.993,c=0.503, md=-150.344,cd=-16.579,step_size_m=-0.150,step_size_c=-0.017,cost=89.628
    >epoch=11, m=5.117,c=0.517, md=-124.668,cd=-14.110,step_size_m=-0.125,step_size_c=-0.014,cost=68.700
    >epoch=12, m=5.221,c=0.530, md=-103.370,cd=-12.063,step_size_m=-0.103,step_size_c=-0.012,cost=54.301
    >epoch=13, m=5.307,c=0.540, md=-85.705,cd=-10.364,step_size_m=-0.086,step_size_c=-0.010,cost=44.393
    >epoch=14, m=5.378,c=0.549, md=-71.053,cd=-8.955,step_size_m=-0.071,step_size_c=-0.009,cost=37.576
    >epoch=15, m=5.436,c=0.557, md=-58.900,cd=-7.786,step_size_m=-0.059,step_size_c=-0.008,cost=32.884
    >epoch=16, m=5.485,c=0.563, md=-48.820,cd=-6.816,step_size_m=-0.049,step_size_c=-0.007,cost=29.655
    >epoch=17, m=5.526,c=0.569, md=-40.459,cd=-6.011,step_size_m=-0.040,step_size_c=-0.006,cost=27.432
    >epoch=18, m=5.559,c=0.575, md=-33.524,cd=-5.344,step_size_m=-0.034,step_size_c=-0.005,cost=25.901
    >epoch=19, m=5.587,c=0.580, md=-27.772,cd=-4.790,step_size_m=-0.028,step_size_c=-0.005,cost=24.846
    >epoch=20, m=5.610,c=0.584, md=-23.001,cd=-4.331,step_size_m=-0.023,step_size_c=-0.004,cost=24.120
    >epoch=21, m=5.629,c=0.588, md=-19.043,cd=-3.949,step_size_m=-0.019,step_size_c=-0.004,cost=23.618
    >epoch=22, m=5.645,c=0.591, md=-15.761,cd=-3.633,step_size_m=-0.016,step_size_c=-0.004,cost=23.272
    >epoch=23, m=5.658,c=0.595, md=-13.039,cd=-3.370,step_size_m=-0.013,step_size_c=-0.003,cost=23.032
    >epoch=24, m=5.669,c=0.598, md=-10.781,cd=-3.153,step_size_m=-0.011,step_size_c=-0.003,cost=22.866
    >epoch=25, m=5.678,c=0.601, md=-8.908,cd=-2.972,step_size_m=-0.009,step_size_c=-0.003,cost=22.750
    >epoch=26, m=5.685,c=0.604, md=-7.354,cd=-2.821,step_size_m=-0.007,step_size_c=-0.003,cost=22.669
    >epoch=27, m=5.691,c=0.606, md=-6.065,cd=-2.697,step_size_m=-0.006,step_size_c=-0.003,cost=22.612
    >epoch=28, m=5.696,c=0.609, md=-4.997,cd=-2.593,step_size_m=-0.005,step_size_c=-0.003,cost=22.571
    >epoch=29, m=5.700,c=0.612, md=-4.110,cd=-2.507,step_size_m=-0.004,step_size_c=-0.003,cost=22.542



```python
## Plot the cost with each iteration
import matplotlib.pyplot as plt
def Simple_LinearRegression_Gradient_Descent_cost_plot(x,y,epochs,learning_rate,n):
    m_curr = 0
    c_curr = 0
    iter = epochs
    cost_val = [] # list to store the cost in each iteration
    intercept = []
    coefficient = []
    for i in range(iter):
        y_pred = (m_curr * x) + c_curr
        cost = 1/n * sum([data**2 for data in (y - y_pred)])
        cost_val.append(cost) ## append the cost_val 
        md = -(2/n) * sum(x * (y - y_pred))
        cd = -(2/n) * sum(y - y_pred)
        m_curr = m_curr - (learning_rate * md)  # 
        coefficient.append(m_curr)
        c_curr = c_curr - (learning_rate * cd)
        intercept.append(c_curr)
    return intercept,coefficient,cost_val
```


```python
x = np.array([3,4,6,12,9,15,10,1,8,13])
y = np.array([16,29,43,65,51,89,57,9,53,68])
epochs = 100
n = len(x)
learning_rate = 0.001
intercept,coefficient,cost_val = Simple_LinearRegression_Gradient_Descent_cost_plot(x,y,epochs,learning_rate,n)

plt.figure(figsize=(5,5))
plt.plot(coefficient, cost_val, '.r',color='blue',label="cost function curve")
plt.xlabel('coefficient')
plt.ylabel('cost_val')
plt.grid(alpha=0.25)
plt.legend()
```


```python
## Now plot grapg to show the regression best fit line 
import matplotlib.pyplot as plt
%matplotlib inline

def Simple_LinearRegression_Gradient_Descent_best_fit_line(x,y,epochs,learning_rate,n):
    m_curr = 0
    c_curr = 0
    iter = epochs
    plt.scatter(x,y,color = "red",label='Scatter Plot')
    for i in range(iter):
        ## y = mx + c
        y_pred = m_curr * x + c_curr
        ## MSE equation == cost_function
        cost = 1/n*sum([val**2 for val in (y - y_pred)])
        ## lets take derivative
        md = -2/n * sum(x*(y - y_pred))
        cd = -2/n * sum(y - y_pred)
        plt.plot(x,y_pred,color = "blue",linewidth='.1') 
        plt.xlabel('Experience')
        plt.ylabel('Salary')
        #plt.style.use('fivethirtyeight')
        #plt.legend()
        m_curr = m_curr - learning_rate * md
        c_curr = c_curr - learning_rate * cd      
```


```python
#x = np.array([3,4,6,12,9,15,10,1,8,13])
#y = np.array([16,29,43,65,51,89,57,9,53,68])
epochs = 100
n = len(x)
learning_rate = 0.001
Simple_LinearRegression_Gradient_Descent_best_fit_line(x,y,epochs,learning_rate,n)
```


```python
lr = LinearRegression()
lr.fit(x.reshape(-1,1),y)
```




    LinearRegression()




```python
lr.intercept_,lr.coef_
```




    (5.334568554790884, array([5.26733722]))




```python
y_predict = lr.predict(x.reshape(-1,1))
y_predict
```




    array([21.1365802 , 26.40391742, 36.93859185, 68.54261514, 52.74060349,
           84.34462679, 58.00794071, 10.60190577, 47.47326628, 73.80995236])




```python
mean_squared_error(y,y_predict)
```




    17.499947061937544




```python
from sklearn.linear_model import ElasticNet,Ridge,Lasso
```


```python
lre = ElasticNet()
lre.fit(x.reshape(-1,1),y)
```




    ElasticNet()




```python
lre.intercept_,lre.coef_
```




    (6.643630737493552, array([5.1057246]))




```python
y_predict_e = lre.predict(x.reshape(-1,1))
y_predict_e
```




    array([21.96080454, 27.06652914, 37.27797834, 67.91232594, 52.59515214,
           83.22949974, 57.70087674, 11.74935534, 47.48942754, 73.01805054])




```python
mean_squared_error(y,y_predict_e)
```




    17.993328121953727



7c59338daaab4eae34d0499c038e1fe09f4535ac
ipynb
Jupyter Notebook
Machine_Learning/Linear_Regression/Gradient_Descent_mathematical_intuition_and_hands-on_Linear_Regression.ipynb
atulpatelDS/Youtube
386204328b3d363d05f65f1f2381f4c2519e30d5
MIT
2021-04-05T17:29:20.000Z
2022-03-16T21:55:44.000Z
Machine_Learning/Linear_Regression/Gradient_Descent_mathematical_intuition_and_hands-on_Linear_Regression.ipynb
MichaelGhaly20/Youtube
386204328b3d363d05f65f1f2381f4c2519e30d5
MIT
Machine_Learning/Linear_Regression/Gradient_Descent_mathematical_intuition_and_hands-on_Linear_Regression.ipynb
MichaelGhaly20/Youtube
386204328b3d363d05f65f1f2381f4c2519e30d5
MIT
2021-05-22T10:40:51.000Z
2022-03-07T01:35:20.000Z
Qwen/Qwen-72B
1. YES
2. YES