```python
#  A regression attempts to fit a of function to observed data to make predictions on new data.
# A linear regression fits a straight line to observed data, attempting to demonstrate a linear relationship
# between variables and make predictions on new data yet to be observed.
```


```python
# Scikit-Learn to perform a basic, unvalidated linear regression on the sample of 10 dogs.
from matplotlib import *
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression

# Import points
df = pd.read_csv('https://bit.ly/3goOAnt', delimiter=",")

# Extract input variables (all rows, all columns but last column)
X = df.values[:, :-1]

# Extract output column (all rows, last column)
Y = df.values[:, -1]

# Fit a line to the points
fit = LinearRegression().fit(X, Y)

# m = 1.7867224, b = -16.51923513
m = fit.coef_.flatten()
b = fit.intercept_.flatten()
print("m = {0}".format(m))
print("b = {0}".format(b))

# show in chart
plt.plot(X, Y, 'o') # scatterplot
plt.plot(X, m*X+b) # line
plt.show()
```


```python
# #  The residual is the numeric difference between the line and the points

#  Another name for residuals are errors, because they reflect how wrong our line is in predicting the data.

# Calculating the residuals for a given line and data

# Import points
points = pd.read_csv('https://bit.ly/3goOAnt', delimiter=",").itertuples()

# Test with a given line
m = 1.93939
b = 4.73333

# calculate sum of squares
for p in points:
    y_actual = p.y
    y_predict = m*p.x + b
    residual = y_actual - y_predict
    print(residual)
```

    -1.67272
    1.3878900000000005
    -0.5515000000000008
    2.5091099999999997
    -0.4302799999999998
    -1.3696699999999993
    0.6909400000000012
    -2.2484499999999983
    2.812160000000002
    -1.1272299999999973



```python
# If we are fitting a straight line through our 10 data points, we likely want to minimize these residuals
# in total so there is as little of a gap as possible between the line and points.
# But how do we measure the “total”? The best approach is to take the sum of squares,
# which simply squares each residual, or multiplies each residual by itself, 
# and sums them. We take each actual y value and subtract from it the predicted y value taken from the line,
# then square and sum all those differences.
```


```python
# You might wonder why we have to square the residuals before summing them.
# Why not just add them up without squaring?
# That will not work because the negatives will cancel out the positives.
# What if we add the absolute values, where we turn all negative values into positive values?
# That sounds promising but absolute values are mathematically inconvenient.
# More specifically, absolute values do not work well with Calculus derivatives 
# which we are going to use later for gradient descent. 
# This is why we choose the squared residuals as our way of totaling the loss.
```


```python
# Calculating the sum of squares for a given line and data

points = pd.read_csv("https://bit.ly/2KF29Bd").itertuples()

# Test with a given line
m = 1.93939
b = 4.73333

sum_of_squares = 0.0

# calculate sum of squares
for p in points:
    y_actual = p.y
    y_predict = m*p.x + b
    residual_squared = (y_predict - y_actual)**2
    sum_of_squares += residual_squared

    
print("sum of squares = {}".format(sum_of_squares))
```

    sum of squares = 28.096969704500005



```python
# Calculating m and b for a simple linear regression

# Load the data
points = list(pd.read_csv('https://bit.ly/2KF29Bd', delimiter=",").itertuples())

n = len(points)

m = (n*sum(p.x*p.y for p in points) - sum(p.x for p in points) *
    sum(p.y for p in points)) / (n*sum(p.x**2 for p in points) -
    sum(p.x for p in points)**2)

b = (sum(p.y for p in points) / n) - m * sum(p.x for p in points) / n

print(m, b)
```

    1.9393939393939394 4.7333333333333325



```python
# Using inverse and transposed matrices to fit a linear regression

import pandas as pd
from numpy.linalg import inv,qr
import numpy as np

# Import points
df = pd.read_csv('https://bit.ly/3goOAnt', delimiter=",")

# Extract input variables (all rows, all columns but last column)
X = df.values[:, :-1].flatten()

# Add placeholder "1" column to generate intercept
X_1 = np.vstack([X, np.ones(len(X))]).T

# Extract output column (all rows, last column)
Y = df.values[:, -1]

# Calculate coefficents for slope and intercept
b = inv(X_1.transpose() @ X_1) @ (X_1.transpose() @ Y)
print(b) # [1.93939394 4.73333333]

# Predict against the y-values
y_predict = X_1.dot(b)

print (y_predict)
```

    [1.93939394 4.73333333]
    [ 6.67272727  8.61212121 10.55151515 12.49090909 14.43030303 16.36969697
     18.30909091 20.24848485 22.18787879 24.12727273]



```python
# Using QR decomposition to perform a linear regression

# Import points
df = pd.read_csv('https://bit.ly/3goOAnt', delimiter=",")

# Extract input variables (all rows, all columns but last column)
X = df.values[:, :-1].flatten()

# Add placeholder "1" column to generate intercept
X_1 = np.vstack([X, np.ones(len(X))]).transpose()

# Extract output column (all rows, last column)
Y = df.values[:, -1]

# calculate coefficents for slope and intercept
# using QR decomposition
Q, R = qr(X_1)
b = inv(R).dot(Q.transpose()).dot(Y)

print(b)

```

    [1.93939394 4.73333333]



```python
# Gradient descent is an optimization technique that uses derivatives and
# iterations to minimize/maximize a set of parameters against an objective.

# Using gradient descent to find the minimum of a parabola

import random


def f(x):
    return (x - 3) ** 2 + 4

def dx_f(x):
    return 2*(x - 3)

# The learning rate
L = 0.001

# The number of iterations to perform gradient descent
iterations = 100_000

 # start at a random x
x = random.randint(-15,15)

for i in range(iterations):

    # get slope
    d_x = dx_f(x)

    # update x by subtracting the (learning rate) * (slope)
    x -= L * d_x

print(x, f(x))
```

    3.000000000000111 4.0



```python
# Performing gradient descent for a linear regression

# Import points from CSV
points = list(pd.read_csv("https://bit.ly/2KF29Bd").itertuples())

# Building the model
m = 0.0
b = 0.0

# The learning Rate
L = .001

# The number of iterations
iterations = 100_000

n = float(len(points))  # Number of elements in X

# Perform Gradient Descent
for i in range(iterations):

    # slope with respect to m
    D_m = sum(2 * p.x * ((m * p.x + b) - p.y) for p in points)

    # slope with respect to b
    D_b = sum(2 * ((m * p.x + b) - p.y) for p in points)

    # update m and b
    m -= L * D_m
    b -= L * D_b

print("y = {0}x + {1}".format(m, b))
```

    y = 1.9393939393939548x + 4.733333333333227



```python
# Calculating partial derivatives for m and b

from sympy import *

m, b, i, n = symbols('m b i n')
x, y = symbols('x y', cls=Function)

sum_of_squares = Sum((m*x(i) + b - y(i)) ** 2, (i, 0, n))

d_m = diff(sum_of_squares, m)
d_b = diff(sum_of_squares, b)
print(d_m)
print(d_b)
```

    Sum(2*(b + m*x(i) - y(i))*x(i), (i, 0, n))
    Sum(2*b + 2*m*x(i) - 2*y(i), (i, 0, n))



```python
# Performing stochastic gradient descent for a linear regression


# Input data
data = pd.read_csv('https://bit.ly/2KF29Bd', header=0)

X = data.iloc[:, 0].values
Y = data.iloc[:, 1].values

n = data.shape[0]  # rows

# Building the model
m = 0.0
b = 0.0

sample_size = 1  # sample size
L = .0001  # The learning Rate
epochs = 1_000_000  # The number of iterations to perform gradient descent

# Performing Stochastic Gradient Descent
for i in range(epochs):
    idx = np.random.choice(n, sample_size, replace=False)
    x_sample = X[idx]
    y_sample = Y[idx]

    # The current predicted value of Y
    Y_pred = m * x_sample + b

    # d/dm derivative of loss function
    D_m = (-2 / sample_size) * sum(x_sample * (y_sample - Y_pred))

    # d/db derivative of loss function
    D_b = (-2 / sample_size) * sum(y_sample - Y_pred)
    m = m - L * D_m  # Update m
    b = b - L * D_b  # Update b

    # print progress
    if i % 10000 == 0:
        print(i, m, b)

print("y = {0}x + {1}".format(m, b))
```

    0 0.006 0.002
    10000 2.36326760682638 1.8835518624103165
    20000 2.208478682839455 2.860757272083633
    30000 2.0990271430034566 3.5015104981731477
    40000 2.059718240314351 3.918670691928773
    50000 2.0254319787495616 4.228272179662717
    60000 2.0128706983709828 4.3987557816635325
    70000 1.9588699155518574 4.511397889311558
    80000 1.9572242882073074 4.573105411192914
    90000 1.9603079930214902 4.646560110817617
    100000 1.915286401953193 4.679197268853813
    110000 1.9403467582673453 4.694118062444541
    120000 1.9599274086298113 4.725943861094068
    130000 1.9368722399981075 4.7143432848607825
    140000 1.963520934610125 4.71478629302077
    150000 1.9360986672130869 4.728305205329085
    160000 1.9324433155776284 4.741555567620757
    170000 1.926229705774608 4.7507334297890695
    180000 1.9432750003962185 4.731457564467172
    190000 1.9652083911680416 4.750695486563088
    200000 1.924857106440728 4.741023799177265
    210000 1.949168382522107 4.742302871466551
    220000 1.9482919821971325 4.732498280457749
    230000 1.9436648255769589 4.744293925905611
    240000 1.9705030140252286 4.742529291867208
    250000 1.9353429418031627 4.739497162984329
    260000 1.9562758398526634 4.738933645814807
    270000 1.9382884118185948 4.728545301878929
    280000 1.93255522206419 4.720959425792094
    290000 1.9410867423424232 4.719485579197047
    300000 1.928348444769958 4.709229998935416
    310000 1.944704859029115 4.734245656426094
    320000 1.9315489900068277 4.7164345649351676
    330000 1.939995611167781 4.709338225607842
    340000 1.961142019124275 4.705429525526002
    350000 1.9216025853518108 4.723954582852459
    360000 1.944900362530313 4.7252435418819125
    370000 1.9399721319259282 4.734938065834019
    380000 1.9497781564660994 4.742384445451385
    390000 1.9805381395468729 4.731931988681261
    400000 1.9454482978428054 4.730618875951833
    410000 1.9703383827976337 4.733665784535996
    420000 1.981650785600547 4.742295189046688
    430000 1.972530745570693 4.74071158972351
    440000 1.9111024810843704 4.726256650760091
    450000 1.9082781943786304 4.731152175860961
    460000 1.9204477114790428 4.739821573687611
    470000 1.9644981269863893 4.745855875675362
    480000 1.9263137432887223 4.743643371721059
    490000 1.91711757641405 4.74294635046912
    500000 1.930934583192176 4.74212184317665
    510000 1.967479133941378 4.736673843592278
    520000 1.9505228605459906 4.728075419769489
    530000 1.9516554472517902 4.728522913632032
    540000 1.9417919849569742 4.749508598760058
    550000 1.9611385059437294 4.766887931996634
    560000 1.9267487723441967 4.745625753029247
    570000 1.9160704416577639 4.735761047329838
    580000 1.929617903629193 4.721682821898547
    590000 1.9149289269251695 4.728326533479536
    600000 1.931097863515108 4.706517870443267
    610000 1.938513500360612 4.72158595083187
    620000 1.9654752270010292 4.727653810731125
    630000 1.9124274374663552 4.714167893880229
    640000 1.9460836587073391 4.721447016138792
    650000 1.9548197285647888 4.714751503819112
    660000 1.946696243788322 4.7234709202203105
    670000 1.943371459630136 4.722904788603818
    680000 1.9573802094954733 4.7256764624636
    690000 1.931115907669294 4.718575794959121
    700000 1.9262178442406925 4.7242553744409905
    710000 1.9584890714298269 4.707751870061937
    720000 1.91096125496631 4.732834014230203
    730000 1.9638450307901556 4.730820586569408
    740000 1.9376318945047226 4.742484193616451
    750000 1.9764575310410655 4.721173739823862
    760000 1.959201460128207 4.728725027441086
    770000 1.9616063379508497 4.745386968448955
    780000 1.9303406405244505 4.7324681138365285
    790000 1.9424228463604172 4.728831976971964
    800000 1.9043294280432401 4.69159602664981
    810000 1.9336889001186668 4.7071650308872695
    820000 1.9441607414771653 4.712772819341543
    830000 1.959822283009035 4.73382719635445
    840000 1.9453503711741207 4.724613103589071
    850000 1.9587792297040645 4.737822839830986
    860000 1.9157005293366143 4.722802115138588
    870000 1.9464146899956396 4.719103639727119
    880000 1.9111757048916815 4.714647715026774
    890000 1.9257806715010586 4.731873835908689
    900000 1.9534173041959169 4.722614847513641
    910000 1.9505340945508731 4.73813717689114
    920000 1.9330475833082221 4.73912605385538
    930000 1.9594103111947985 4.752444875710609
    940000 1.9415876754302852 4.754499178310029
    950000 1.9306788443386962 4.752825434666956
    960000 1.921548945614283 4.76041767073736
    970000 1.962487828671824 4.772474403778124
    980000 1.9865264430013605 4.754946770865421
    990000 1.9421834301828678 4.7535420839260265
    y = 1.9502363279513883x + 4.748196684882565



```python
# correlation coefficient, also called the Pearson correlation,
# which measures the strength of the relationship between two variables as a value 
# between -1 and 1. A correlation coefficient closer to 0 indicates there is no correlation.
# A correlation coefficient closer to 1 indicates a strong positive correlation,
# meaning when one variable increases the other proportionally increases.
# If it is closer to -1 then it indicates a strong negative correlation,
# which means as one variable increases the other proportionally decreases.
```


```python
# Using Pandas to see the correlation coefficent between every pair of variables

# Read data into Pandas dataframe
df = pd.read_csv('https://bit.ly/2KF29Bd', delimiter=",")

# Print correlations between variables
correlations = df.corr(method='pearson')
print(correlations)

```

              x         y
    x  1.000000  0.957586
    y  0.957586  1.000000



```python
# Calculating correlation coefficient from scratch in Python

from math import sqrt

# Import points from CSV
points = list(pd.read_csv("https://bit.ly/2KF29Bd").itertuples())
n = len(points)

numerator = n * sum(p.x * p.y for p in points) - \
            sum(p.x for p in points) * sum(p.y for p in points)

denominator = sqrt(n*sum(p.x**2 for p in points) - sum(p.x for p in points)**2) \
              * sqrt(n*sum(p.y**2 for p in points) - sum(p.y for p in points)**2)

corr = numerator / denominator

print(corr)
```

    0.9575860952087218



```python
#  Calculating the critical value from a T-distribution

from scipy.stats import t

n = 10
lower_cv = t(n-1).ppf(.025)
upper_cv = t(n-1).ppf(.975)

print(lower_cv, upper_cv)
```

    -2.262157162740992 2.2621571627409915



```python
# Testing significance for linear-looking data

from scipy.stats import t
from math import sqrt

# sample size
n = 10

lower_cv = t(n-1).ppf(.025)
upper_cv = t(n-1).ppf(.975)

# correlation coefficient
# derived from data https://bit.ly/2KF29Bd
r = 0.957586

# Perform the test
test_value = r / sqrt((1-r**2) / (n-2))

print("TEST VALUE: {}".format(test_value))
print("CRITICAL RANGE: {}, {}".format(lower_cv, upper_cv))

if test_value < lower_cv or test_value > upper_cv:
    print("CORRELATION PROVEN, REJECT H0")
else:
    print("CORRELATION NOT PROVEN, FAILED TO REJECT H0 ")

# Calculate p-value
if test_value > 0:
    p_value = 1.0 - t(n-1).cdf(test_value)
else:
    p_value = t(n-1).cdf(test_value)

# Two-tailed, so multiply by 2
p_value = p_value * 2
print("P-VALUE: {}".format(p_value))
```

    TEST VALUE: 9.399564671312076
    CRITICAL RANGE: -2.262157162740992, 2.2621571627409915
    CORRELATION PROVEN, REJECT H0
    P-VALUE: 5.9763860877914965e-06



```python
# Creating a correlation matrix in Pandas

# Read data into Pandas dataframe
df = pd.read_csv('https://bit.ly/2KF29Bd', delimiter=",")

# Print correlations between variables
coeff_determination = df.corr(method='pearson') ** 2
print(coeff_determination)
```

              x         y
    x  1.000000  0.916971
    y  0.916971  1.000000



```python
# Calculating a prediction interval of vet visits for a dog that’s 8.5 years old

from math import sqrt

# Load the data
points = list(pd.read_csv('https://bit.ly/2KF29Bd', delimiter=",").itertuples())

n = len(points)

# Linear Regression Line
m = 1.939
b = 4.733

# Calculate Prediction Interval for x = 8.5
x_0 = 8.5
x_mean = sum(p.x for p in points) / len(points)

t_value = t(n - 2).ppf(.975)

standard_error = sqrt(sum((p.y - (m * p.x + b)) ** 2 for p in points) / (n - 2))

margin_of_error = t_value * standard_error * \
                  sqrt(1 + (1 / n) + (n * (x_0 - x_mean) ** 2) / \
                       (n * sum(p.x ** 2 for p in points) - sum(p.x for p in points) ** 2))

predicted_y = m*x_0 + b

# Calculate prediction interval
print(predicted_y - margin_of_error, predicted_y + margin_of_error)
```

    16.462516875955465 25.966483124044537



```python
# Doing a train/test split on linear regression

from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split

# Load the data
df = pd.read_csv('https://bit.ly/3cIH97A', delimiter=",")

# Extract input variables (all rows, all columns but last column)
X = df.values[:, :-1]

# Extract output column (all rows, last column)
Y = df.values[:, -1]

# Separate training and testing data
# This leaves a third of the data out for testing
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=1/3)

model = LinearRegression()
model.fit(X_train, Y_train)
result = model.score(X_test, Y_test)
print("R^2: %.3f" % result)
```

    R^2: 0.993



```python
# Using 3-fold cross validation for a linear regression


from sklearn.linear_model import LinearRegression
from sklearn.model_selection import KFold, cross_val_score

df = pd.read_csv('https://bit.ly/3cIH97A', delimiter=",")

# Extract input variables (all rows, all columns but last column)
X = df.values[:, :-1]

# Extract output column (all rows, last column)\
Y = df.values[:, -1]

# Perform a simple linear regression
kfold = KFold(n_splits=3, random_state=7, shuffle=True)
model = LinearRegression()
results = cross_val_score(model, X, Y, cv=kfold)
print(results)
print("MSE: mean=%.3f (stdev-%.3f)" % (results.mean(), results.std()))
```

    [0.99337354 0.99345032 0.99251425]
    MSE: mean=0.993 (stdev-0.000)



```python
# Using a random-fold validation for a linear regression


from sklearn.linear_model import LinearRegression
from sklearn.model_selection import KFold, cross_val_score

df = pd.read_csv('https://bit.ly/3cIH97A', delimiter=",")

# Extract input variables (all rows, all columns but last column)
X = df.values[:, :-1]

# Extract output column (all rows, last column)\
Y = df.values[:, -1]

# Perform a simple linear regression
kfold = KFold(n_splits=3, random_state=7, shuffle=True)
model = LinearRegression()
results = cross_val_score(model, X, Y, cv=kfold)
print(results)
print("MSE: mean=%.3f (stdev-%.3f)" % (results.mean(), results.std()))
```

    [0.99337354 0.99345032 0.99251425]
    MSE: mean=0.993 (stdev-0.000)



```python
#  A linear regressoin with two input variables

# Load the data
df = pd.read_csv('https://bit.ly/2X1HWH7', delimiter=",")

# Extract input variables (all rows, all columns but last column)
X = df.values[:, :-1]

# Extract output column (all rows, last column)\
Y = df.values[:, -1]

# Training
fit = LinearRegression().fit(X, Y)

# Print coefficients
print("Coefficients = {0}".format(fit.coef_))
print("Intercept = {0}".format(fit.intercept_))
print("z = {0} + {1}x + {2}y".format(fit.intercept_, fit.coef_[0], fit.coef_[1]))
```

    Coefficients = [2.00672647 3.00203798]
    Intercept = 20.109432820035963
    z = 20.109432820035963 + 2.0067264725128062x + 3.002037976646693y



```python

```

5272cf2d52311bb77a499409d367cbec74255816
ipynb
Jupyter Notebook
Maths/Linear Regression/Linear Regression.ipynb
rishi9504/Data-Science
10344bf641c601bf16451ddd9eaa28ab4c0fc75b
MIT
Maths/Linear Regression/Linear Regression.ipynb
rishi9504/Data-Science
10344bf641c601bf16451ddd9eaa28ab4c0fc75b
MIT
Maths/Linear Regression/Linear Regression.ipynb
rishi9504/Data-Science
10344bf641c601bf16451ddd9eaa28ab4c0fc75b
MIT
Qwen/Qwen-72B
1. YES
2. YES