<kaggle_start><code>import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
from functools import partial

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
import os

for dirname, _, filenames in os.walk("/kaggle/input"):
    for filename in filenames:
        print(os.path.join(dirname, filename))


# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All"
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
class Gradient_Decent:
    """
    Calculates the local minimum
    accuracy, how close you want the output x to be to the proper value
    Lambda, the gradient decent rate
    """

    def __init__(self, grad_func, func=None, Lambda=0.1, accuracy=0.001):
        self.func = func
        self.grad_func = grad_func
        self.Lambda = Lambda
        self.accuracy = accuracy

    def sample(start, end):
        pass

    def start(self, x_0, iterations=3000):
        x_k = x_0
        for k in range(iterations):
            grad_val = self.grad_func(x_k)
            x_k = x_k - self.Lambda * grad_val
            if np.linalg.norm(self.grad_func(x_k)) < self.accuracy:
                # Found solution
                print("Yes")
                return x_k
        print("No")
        return x_k


# Easy Tests
def f(x):
    x_1 = x[0]
    x_2 = x[1]
    return (x_1 - 3) ** 2 + y_2**2


def grad_f(x):
    x_1 = x[0]
    x_2 = x[1]
    return np.array([2 * (x_1 - 3), 2 * x_2])


test_gradient_decent = Gradient_Decent(grad_func=grad_f)
result = test_gradient_decent.start(np.array([5, 4]))
assert max(abs(result - np.array([3, 0]))) < 1e-2
result = test_gradient_decent.start(np.array([-50, 23]))
assert max(abs(result - np.array([3, 0]))) < 1e-2


# Harder Tests
def grad_f(x):
    x_1 = x[0]
    x_2 = x[1]
    x_3 = x[2]
    y_1 = 2 * x_2**2 * x_3**2 * x_1 + 2 * x_1 * x_3**2
    y_2 = 2 * x_2 * x_3**2 * x_1**2
    y_3 = 2 * x_2**2 * x_3 * x_1**2 + 2 * x_3 * x_1**2
    return np.array([y_1, y_2, y_3])


test_gradient_decent = Gradient_Decent(grad_func=grad_f, Lambda=0.0001)
result = test_gradient_decent.start(np.array([5, 4, 10]))
assert abs(result[0]) < 1e-2
a = np.array(result)
grad_f(a)
result
9.9e-4


# Neural Net Section
def sigmoid(x):
    if x > 700:
        return 1
    if x < -700:
        return 0
    return 1 / (1 + np.e ** (-x))


def dsigmoid(x):
    if (x < -700) or (700 < x):
        return 0
    return np.e ** (-x) * sigmoid(x) ** 2


assert dsigmoid(0) == 1 / 4


class BasicNeuralNet:
    """
    1 layer neural network
    """

    def __init__(self):
        pass

    def fit(self, X, y):
        """X,y pandas data frames"""
        X = X.assign(_BIAS_=[1] * len(X))  # Add bias weight
        num_X = X.to_numpy()
        self.n = len(num_X[0])
        self.a = np.zeros(self.n)
        self.my_grad_MSE = partial(grad_MSE, x=num_X, y=y.to_numpy())
        my_gradient_decent = Gradient_Decent(grad_func=self.my_grad_MSE, Lambda=0.1)
        # partail function
        # use gradient decent
        new_a = my_gradient_decent.start(x_0=self.a, iterations=7000)
        self.a = new_a
        # get the weights and make a predict function

    def predict(self, X):
        """X is 2D array"""
        m = len(X)  # records
        X = np.append(X, np.array([[1]] * m), axis=1)
        y = np.zeros(m)
        for record_id in range(len(X)):
            y[record_id] = sigmoid(sum(self.a * X[record_id]))
        return y

    def MSE(self, x):
        """The cost function"""
        pass


def grad_MSE(a, x, y):
    """x is a 2D numpy array
    a are the weights
    y is the actual value"""
    n = len(x[0])
    m = len(x)
    grad = np.zeros(n)
    for grad_position in range(n):
        total = 0
        for i in range(m):
            # i is the record ID
            total += (
                2
                * dsigmoid(sum(a * x[i]))
                * x[i][grad_position]
                * (sigmoid(sum(a * x[i])) - y[i])
            )
        grad[grad_position] = total
    return grad


# Test
test_X = [[1, 2, 3, 1], [3, 2, 1, 0], [2, 2, 2, 0]]
test_Xdf = pd.DataFrame(data=test_X)
test_y = [0, 1, 0]
test_ydf = pd.DataFrame(data=test_y)
test_NeuralNet = BasicNeuralNet()
test_NeuralNet.fit(X=test_Xdf, y=test_ydf)
print(test_NeuralNet.my_grad_MSE(test_NeuralNet.a))
assert max(abs(test_NeuralNet.predict(test_X) - [0, 1, 0])) < 0.1
test_NeuralNet.predict(test_X)
test_ydf
test_a = np.array([0, 0, 3, 0, 0])
test_x = np.array([[1, 1, 3, 4, 5], [5, 4, 3, 2, 1], [1, 2, 3, 4, 5]])
test_y = np.array([1, 0, 1])
grad_MSE(test_a, test_x, test_y)
titanic_file_path = "../input/titanic/train.csv"
titanic_data = pd.read_csv(titanic_file_path)
titanic_data.head()
y = titanic_data.Survived
feature_columns = ["Pclass", "Sex", "Age", "Fare"]  # , "Cabin"]
X = titanic_data[feature_columns]
new_columns = ["Pclass", "isMale", "isFemale", "Age", "Fare"]
# is_First = (X["Pclass"]==1)
# is_Second = (X["Pclass"]==2)
# is_Third = (X["Pclass"]==3)
is_Male = X["Sex"] == "male"
is_Female = X["Sex"] == "female"
the_data = [X["Pclass"], is_Male, is_Female, X["Age"], X["Fare"]]
new_X = pd.concat(the_data, axis=1, keys=new_columns)
# Remove NAN and change to 0
new_X = new_X.replace(np.nan, 0)
new_X
titanicNeuralNet = BasicNeuralNet()
X_train = new_X[0:800]
y_train = y[0:800]
titanicNeuralNet.fit(X_train, y_train)
titanicNeuralNet.a
# Mean average error
sum(abs(titanicNeuralNet.predict(new_X[0:10]) - y[0:10]))
# Number wrong
sum(abs(np.round(titanicNeuralNet.predict(new_X[0:880])) - y[0:880]))
# Number incorrect
sum((np.round(titanicNeuralNet.predict(new_X[0:300])).astype(np.int) - y[0:300]) ** 2)
# now for test data
# path to file you will use for predictions
test_data_path = "../input/titanic/test.csv"
# read test data file using pandas
test_data = pd.read_csv(test_data_path)
# create test_X which comes from test_data but includes only the columns you used for prediction.
# The list of columns is stored in a variable called features
feature_columns = ["Pclass", "Sex", "Age", "Fare"]
test_X = test_data[feature_columns]
new_columns = ["Pclass", "isMale", "isFemale", "Age", "Fare"]
is_Male = test_X["Sex"] == "male"
is_Female = test_X["Sex"] == "female"
the_data = [test_X["Pclass"], is_Male, is_Female, test_X["Age"], test_X["Fare"]]
test_new_X = pd.concat(the_data, axis=1, keys=new_columns)
# Remove NAN and change to 0
test_new_X = test_new_X.replace(np.nan, 0)
# make predictions which we will submit.
test_preds = np.round(titanicNeuralNet.predict(test_new_X)).astype(np.int)
# The lines below shows how to save predictions in format used for competition scoring
# Just uncomment them.
output = pd.DataFrame({"PassengerId": test_data.PassengerId, "Survived": test_preds})
output.to_csv("submission.csv", index=False)
output[0:20]
<|endoftext|>
[{"Id": 45894332, "ScriptId": 12596012, "ParentScriptVersionId": NaN, "ScriptLanguageId": 9, "AuthorUserId": 6065461, "CreationDate": "10/31/2020 15:28:39", "VersionNumber": 2.0, "Title": "Gradient Decent", "EvaluationDate": "10/31/2020", "IsChange": true, "TotalLines": 256.0, "LinesInsertedFromPrevious": 7.0, "LinesChangedFromPrevious": 0.0, "LinesUnchangedFromPrevious": 249.0, "LinesInsertedFromFork": NaN, "LinesDeletedFromFork": NaN, "LinesChangedFromFork": NaN, "LinesUnchangedFromFork": NaN, "TotalVotes": 0}]
import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)
from functools import partial

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
import os

for dirname, _, filenames in os.walk("/kaggle/input"):
    for filename in filenames:
        print(os.path.join(dirname, filename))


# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All"
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
class Gradient_Decent:
    """
    Calculates the local minimum
    accuracy, how close you want the output x to be to the proper value
    Lambda, the gradient decent rate
    """

    def __init__(self, grad_func, func=None, Lambda=0.1, accuracy=0.001):
        self.func = func
        self.grad_func = grad_func
        self.Lambda = Lambda
        self.accuracy = accuracy

    def sample(start, end):
        pass

    def start(self, x_0, iterations=3000):
        x_k = x_0
        for k in range(iterations):
            grad_val = self.grad_func(x_k)
            x_k = x_k - self.Lambda * grad_val
            if np.linalg.norm(self.grad_func(x_k)) < self.accuracy:
                # Found solution
                print("Yes")
                return x_k
        print("No")
        return x_k


# Easy Tests
def f(x):
    x_1 = x[0]
    x_2 = x[1]
    return (x_1 - 3) ** 2 + y_2**2


def grad_f(x):
    x_1 = x[0]
    x_2 = x[1]
    return np.array([2 * (x_1 - 3), 2 * x_2])


test_gradient_decent = Gradient_Decent(grad_func=grad_f)
result = test_gradient_decent.start(np.array([5, 4]))
assert max(abs(result - np.array([3, 0]))) < 1e-2
result = test_gradient_decent.start(np.array([-50, 23]))
assert max(abs(result - np.array([3, 0]))) < 1e-2


# Harder Tests
def grad_f(x):
    x_1 = x[0]
    x_2 = x[1]
    x_3 = x[2]
    y_1 = 2 * x_2**2 * x_3**2 * x_1 + 2 * x_1 * x_3**2
    y_2 = 2 * x_2 * x_3**2 * x_1**2
    y_3 = 2 * x_2**2 * x_3 * x_1**2 + 2 * x_3 * x_1**2
    return np.array([y_1, y_2, y_3])


test_gradient_decent = Gradient_Decent(grad_func=grad_f, Lambda=0.0001)
result = test_gradient_decent.start(np.array([5, 4, 10]))
assert abs(result[0]) < 1e-2
a = np.array(result)
grad_f(a)
result
9.9e-4


# Neural Net Section
def sigmoid(x):
    if x > 700:
        return 1
    if x < -700:
        return 0
    return 1 / (1 + np.e ** (-x))


def dsigmoid(x):
    if (x < -700) or (700 < x):
        return 0
    return np.e ** (-x) * sigmoid(x) ** 2


assert dsigmoid(0) == 1 / 4


class BasicNeuralNet:
    """
    1 layer neural network
    """

    def __init__(self):
        pass

    def fit(self, X, y):
        """X,y pandas data frames"""
        X = X.assign(_BIAS_=[1] * len(X))  # Add bias weight
        num_X = X.to_numpy()
        self.n = len(num_X[0])
        self.a = np.zeros(self.n)
        self.my_grad_MSE = partial(grad_MSE, x=num_X, y=y.to_numpy())
        my_gradient_decent = Gradient_Decent(grad_func=self.my_grad_MSE, Lambda=0.1)
        # partail function
        # use gradient decent
        new_a = my_gradient_decent.start(x_0=self.a, iterations=7000)
        self.a = new_a
        # get the weights and make a predict function

    def predict(self, X):
        """X is 2D array"""
        m = len(X)  # records
        X = np.append(X, np.array([[1]] * m), axis=1)
        y = np.zeros(m)
        for record_id in range(len(X)):
            y[record_id] = sigmoid(sum(self.a * X[record_id]))
        return y

    def MSE(self, x):
        """The cost function"""
        pass


def grad_MSE(a, x, y):
    """x is a 2D numpy array
    a are the weights
    y is the actual value"""
    n = len(x[0])
    m = len(x)
    grad = np.zeros(n)
    for grad_position in range(n):
        total = 0
        for i in range(m):
            # i is the record ID
            total += (
                2
                * dsigmoid(sum(a * x[i]))
                * x[i][grad_position]
                * (sigmoid(sum(a * x[i])) - y[i])
            )
        grad[grad_position] = total
    return grad


# Test
test_X = [[1, 2, 3, 1], [3, 2, 1, 0], [2, 2, 2, 0]]
test_Xdf = pd.DataFrame(data=test_X)
test_y = [0, 1, 0]
test_ydf = pd.DataFrame(data=test_y)
test_NeuralNet = BasicNeuralNet()
test_NeuralNet.fit(X=test_Xdf, y=test_ydf)
print(test_NeuralNet.my_grad_MSE(test_NeuralNet.a))
assert max(abs(test_NeuralNet.predict(test_X) - [0, 1, 0])) < 0.1
test_NeuralNet.predict(test_X)
test_ydf
test_a = np.array([0, 0, 3, 0, 0])
test_x = np.array([[1, 1, 3, 4, 5], [5, 4, 3, 2, 1], [1, 2, 3, 4, 5]])
test_y = np.array([1, 0, 1])
grad_MSE(test_a, test_x, test_y)
titanic_file_path = "../input/titanic/train.csv"
titanic_data = pd.read_csv(titanic_file_path)
titanic_data.head()
y = titanic_data.Survived
feature_columns = ["Pclass", "Sex", "Age", "Fare"]  # , "Cabin"]
X = titanic_data[feature_columns]
new_columns = ["Pclass", "isMale", "isFemale", "Age", "Fare"]
# is_First = (X["Pclass"]==1)
# is_Second = (X["Pclass"]==2)
# is_Third = (X["Pclass"]==3)
is_Male = X["Sex"] == "male"
is_Female = X["Sex"] == "female"
the_data = [X["Pclass"], is_Male, is_Female, X["Age"], X["Fare"]]
new_X = pd.concat(the_data, axis=1, keys=new_columns)
# Remove NAN and change to 0
new_X = new_X.replace(np.nan, 0)
new_X
titanicNeuralNet = BasicNeuralNet()
X_train = new_X[0:800]
y_train = y[0:800]
titanicNeuralNet.fit(X_train, y_train)
titanicNeuralNet.a
# Mean average error
sum(abs(titanicNeuralNet.predict(new_X[0:10]) - y[0:10]))
# Number wrong
sum(abs(np.round(titanicNeuralNet.predict(new_X[0:880])) - y[0:880]))
# Number incorrect
sum((np.round(titanicNeuralNet.predict(new_X[0:300])).astype(np.int) - y[0:300]) ** 2)
# now for test data
# path to file you will use for predictions
test_data_path = "../input/titanic/test.csv"
# read test data file using pandas
test_data = pd.read_csv(test_data_path)
# create test_X which comes from test_data but includes only the columns you used for prediction.
# The list of columns is stored in a variable called features
feature_columns = ["Pclass", "Sex", "Age", "Fare"]
test_X = test_data[feature_columns]
new_columns = ["Pclass", "isMale", "isFemale", "Age", "Fare"]
is_Male = test_X["Sex"] == "male"
is_Female = test_X["Sex"] == "female"
the_data = [test_X["Pclass"], is_Male, is_Female, test_X["Age"], test_X["Fare"]]
test_new_X = pd.concat(the_data, axis=1, keys=new_columns)
# Remove NAN and change to 0
test_new_X = test_new_X.replace(np.nan, 0)
# make predictions which we will submit.
test_preds = np.round(titanicNeuralNet.predict(test_new_X)).astype(np.int)
# The lines below shows how to save predictions in format used for competition scoring
# Just uncomment them.
output = pd.DataFrame({"PassengerId": test_data.PassengerId, "Survived": test_preds})
output.to_csv("submission.csv", index=False)
output[0:20]
