<kaggle_start><code># # @author Julien WUTHRICH
import pandas as pd
import numpy as np

df = pd.read_csv("../input/train_ver2.csv", nrows=100000)
import time


def timeit(method):
    def timed(*args, **kw):
        ts = time.time()
        result = method(*args, **kw)
        te = time.time()
        print("Duration = {}".format(te - ts))
        return result

    return timed


cols = {
    "fecha_dato": "Day",
    "ncodpers": "Id",
    "ind_empleado": "Employee",
    "pais_residenca": "Country",
    "sexo": "Sexe",
    "cod_prov": "CountryId",
    "age": "Age",
    "fecha_alta": "RegistrationDate",
    "ind_nuevo": "NewCustomer",
    "antiguedad": "Seniority",
    "indrel": "Premium",
    "ult_fec_cli_1t": "EndPremium",
    "indrel_1mes": "CustomerType",
    "tiprel_1mes": "RelationType",
    "indresi": "CountryBank",
    "indext": "CountryBirth",
    "conyuemp": "Spouse",
    "canal_entrada": "Channel",
    "indfall": "Deceased",
    "tipodom": "PrimaryAdress",
    "nomprov": "ProvinceName",
    "ind_actividad_cliente": "CustomerActivity",
    "renta": "Income",
    "segmento": "Job",
}
df = df.rename(columns=cols)
from collections import namedtuple

TRAPEZE = namedtuple("SubCategory", ["x1", "x2", "x3", "x4"])


class Tools(object):
    """All tools needed for this exp."""

    def __init__(self):
        """Constructor."""
        pass

    def count_frequency(df, col, new_col="Freq"):
        """Count the occurence of a value, in a column.
        :params df: train dataset
        :type df: dataframe
        :params col: column to work on
        :type col: str or list
        :return: dataframe with new col, Frequency
        :rtype: dataframe
        """
        df[new_col] = df.groupby(col)[col].transform("count")
        return df

    def change_nan_value(df, new_value):
        """Change the nan into new value.
        :params df: dataset
        :params new_val: value to change from NaN
        :return: df or serie without NaN
        """
        return df.where((pd.notnull(df)), new_value)

    def pairwise(data, last=False):
        """Generator for pair data.
        :rparams data: dataframe or list
        :params last: pair btw last and first ?
        :return: generator of pair
        :rtype: list
        """
        res = None
        if isinstance(data, pd.DataFrame):
            res = [(x[1], y[1]) for x, y in zip(data.iterrows(), data[1:].iterrows())]
            if last:
                res = res + [
                    (x[1], y[1])
                    for x, y in zip(data[-1:].iterrows(), data[:1].iterrows())
                ]
        elif isinstance(data, list):
            res = [(x, y) for x, y in zip(data, data[1:])]
            if last:
                res = res + [(x, y) for x, y in zip(data[-1:], data[:1])]
        return res

    def select_cols_after_one(df, col):
        """Select cols after the one given.
        :params df: dataset
        :params col: after this one
        :return: subset df
        """
        return df.ix[:, col:]

    def remove_rows_contains_null(df, col):
        """
        Remove all rows which contain a None in the dataframe column
        """
        return df[df[col].notnull()]


class Fuzzy(object):
    """Generate the fuzzy value."""

    def __init__(self, df):
        """Constructor."""
        self.df = df
        self.remove_row_without_age

    @property
    def remove_row_without_age(self):
        """."""
        self.df = self.df[self.df["Age"] != " NA"]

    def belong_line(self, pt1, pt2, value):
        """Belong line btw the pt1 and pt2.
        :params pt1: (x1, y1)
        :params pt2: (x2, y2)
        """
        x1 = pt1[0]
        x2 = pt2[0]
        contains = range(x1, x2)
        if value in contains:
            return 1
        else:
            return 0

    def belong_fall(self, pt1, pt2, value):
        """Find slope of the line and calc the ordonate of the value.
        :params pt1: (x1, y1)
        :params pt2: (x2, y2)
        """
        if value > pt1[0] and value > pt2[0]:
            return 0
        elif value < pt1[0] and value < pt2[0]:
            return 0
        alpha = (pt2[1] - pt1[1]) / (pt2[0] - pt1[0])
        b = pt1[1] - alpha * pt1[0]
        y = value * alpha + b
        y = abs(round(y, 2))
        if 0 <= y <= 1:
            return y
        else:
            return 0

    def function_membership(self, value, trapeze):
        """Belong of a value.
        :params value: value to define the origin
        :params trapeze: the value of the behaviour
        """
        left_bot = (trapeze.x1, 0.0)
        left_top = (trapeze.x2, 1.0)
        right_top = (trapeze.x3, 1.0)
        right_bot = (trapeze.x4, 0.0)
        res1 = self.belong_fall(left_bot, left_top, value)
        res2 = self.belong_line(left_top, right_top, value)
        res3 = self.belong_fall(right_top, right_bot, value)
        val = [x for x in [res1, res2, res3] if 0 < x <= 1]
        try:
            return val[0]
        except:
            return 0

    @timeit
    def main(self):
        """Compute the new cols with values."""
        kid = TRAPEZE(0, 3, 18, 22)
        student = TRAPEZE(18, 22, 26, 28)
        worker = TRAPEZE(26, 28, 55, 60)
        retired = TRAPEZE(55, 60, 95, 120)
        lst_first, lst_second, lst_third, lst_last, lst_id = [], [], [], [], []
        for idx, row in self.df.iterrows():
            age = int(row["Age"])
            lst_first.append(self.function_membership(age, kid))
            lst_second.append(self.function_membership(age, student))
            lst_third.append(self.function_membership(age, worker))
            lst_last.append(self.function_membership(age, retired))
            lst_id.append(row["Id"])
        df_age = pd.DataFrame([lst_first, lst_second, lst_third, lst_last, lst_id]).T
        df_age.columns = ["Kid", "Student", "Worker", "Retired", "Id"]
        return pd.merge(self.df, df_age, on="Id")


df = Fuzzy(df).main()


class TransformCols(object):
    """Transform cols qualitative to quantitative."""

    def __init__(self, df):
        """Constructor."""
        self.df = df

    @timeit
    def main(self):
        """Create for sexe and job for now."""
        self.df["_Sexe"] = self.df["Sexe"].apply(lambda x: 1 if x == "H" else 0)
        self.df["Job"] = Tools.change_nan_value(self.df["Job"], "05")
        self.df["_Job"] = self.df["Job"].apply(lambda x: x[:2])
        self.df["_Job"] = self.df["_Job"].astype(int)
        self.df["Seniority"] = self.df["Seniority"].astype(int)
        self.df["Age"] = self.df["Age"].astype(int)
        return self.df


df = TransformCols(df).main()
df.head()
available_cols = [
    "Age",
    "NewCustomer",
    "Seniority",
    "Premium",
    "CustomerType",
    "CountryId",
    "CustomerActivity",
    "Income",
    "Kid",
    "Student",
    "Worker",
    "Retired",
    "_Sexe",
    "_Job",
]
cols_to_pred = [
    "ind_cco_fin_ult1",
    "ind_recibo_ult1",
    "ind_cder_fin_ult1",
    "ind_cno_fin_ult1",
    "ind_ctju_fin_ult1",
    "ind_ctma_fin_ult1",
    "ind_ctop_fin_ult1",
    "ind_ctpp_fin_ult1",
    "ind_deco_fin_ult1",
    "ind_deme_fin_ult1",
    "ind_dela_fin_ult1",
    "ind_ecue_fin_ult1",
    "ind_fond_fin_ult1",
    "ind_hip_fin_ult1",
    "ind_plan_fin_ult1",
    "ind_pres_fin_ult1",
    "ind_reca_fin_ult1",
    "ind_tjcr_fin_ult1",
    "ind_valo_fin_ult1",
    "ind_viv_fin_ult1",
    "ind_nomina_ult1",
    "ind_nom_pens_ult1",
]
df = df.drop(["Spouse", "PrimaryAdress"], 1)
df_corr = df.corr(method="spearman")
<|endoftext|>
[{"Id": 469856, "ScriptId": 123325, "ParentScriptVersionId": NaN, "ScriptLanguageId": 9, "AuthorUserId": 576054, "CreationDate": "11/27/2016 16:45:54", "VersionNumber": 8.0, "Title": "Create new columns based on fuzzy logic", "EvaluationDate": "11/27/2016", "IsChange": true, "TotalLines": NaN, "LinesInsertedFromPrevious": 0.0, "LinesChangedFromPrevious": 2.0, "LinesUnchangedFromPrevious": 439.0, "LinesInsertedFromFork": NaN, "LinesDeletedFromFork": NaN, "LinesChangedFromFork": NaN, "LinesUnchangedFromFork": NaN, "TotalVotes": 0}]
# # @author Julien WUTHRICH
import pandas as pd
import numpy as np

df = pd.read_csv("../input/train_ver2.csv", nrows=100000)
import time


def timeit(method):
    def timed(*args, **kw):
        ts = time.time()
        result = method(*args, **kw)
        te = time.time()
        print("Duration = {}".format(te - ts))
        return result

    return timed


cols = {
    "fecha_dato": "Day",
    "ncodpers": "Id",
    "ind_empleado": "Employee",
    "pais_residenca": "Country",
    "sexo": "Sexe",
    "cod_prov": "CountryId",
    "age": "Age",
    "fecha_alta": "RegistrationDate",
    "ind_nuevo": "NewCustomer",
    "antiguedad": "Seniority",
    "indrel": "Premium",
    "ult_fec_cli_1t": "EndPremium",
    "indrel_1mes": "CustomerType",
    "tiprel_1mes": "RelationType",
    "indresi": "CountryBank",
    "indext": "CountryBirth",
    "conyuemp": "Spouse",
    "canal_entrada": "Channel",
    "indfall": "Deceased",
    "tipodom": "PrimaryAdress",
    "nomprov": "ProvinceName",
    "ind_actividad_cliente": "CustomerActivity",
    "renta": "Income",
    "segmento": "Job",
}
df = df.rename(columns=cols)
from collections import namedtuple

TRAPEZE = namedtuple("SubCategory", ["x1", "x2", "x3", "x4"])


class Tools(object):
    """All tools needed for this exp."""

    def __init__(self):
        """Constructor."""
        pass

    def count_frequency(df, col, new_col="Freq"):
        """Count the occurence of a value, in a column.
        :params df: train dataset
        :type df: dataframe
        :params col: column to work on
        :type col: str or list
        :return: dataframe with new col, Frequency
        :rtype: dataframe
        """
        df[new_col] = df.groupby(col)[col].transform("count")
        return df

    def change_nan_value(df, new_value):
        """Change the nan into new value.
        :params df: dataset
        :params new_val: value to change from NaN
        :return: df or serie without NaN
        """
        return df.where((pd.notnull(df)), new_value)

    def pairwise(data, last=False):
        """Generator for pair data.
        :rparams data: dataframe or list
        :params last: pair btw last and first ?
        :return: generator of pair
        :rtype: list
        """
        res = None
        if isinstance(data, pd.DataFrame):
            res = [(x[1], y[1]) for x, y in zip(data.iterrows(), data[1:].iterrows())]
            if last:
                res = res + [
                    (x[1], y[1])
                    for x, y in zip(data[-1:].iterrows(), data[:1].iterrows())
                ]
        elif isinstance(data, list):
            res = [(x, y) for x, y in zip(data, data[1:])]
            if last:
                res = res + [(x, y) for x, y in zip(data[-1:], data[:1])]
        return res

    def select_cols_after_one(df, col):
        """Select cols after the one given.
        :params df: dataset
        :params col: after this one
        :return: subset df
        """
        return df.ix[:, col:]

    def remove_rows_contains_null(df, col):
        """
        Remove all rows which contain a None in the dataframe column
        """
        return df[df[col].notnull()]


class Fuzzy(object):
    """Generate the fuzzy value."""

    def __init__(self, df):
        """Constructor."""
        self.df = df
        self.remove_row_without_age

    @property
    def remove_row_without_age(self):
        """."""
        self.df = self.df[self.df["Age"] != " NA"]

    def belong_line(self, pt1, pt2, value):
        """Belong line btw the pt1 and pt2.
        :params pt1: (x1, y1)
        :params pt2: (x2, y2)
        """
        x1 = pt1[0]
        x2 = pt2[0]
        contains = range(x1, x2)
        if value in contains:
            return 1
        else:
            return 0

    def belong_fall(self, pt1, pt2, value):
        """Find slope of the line and calc the ordonate of the value.
        :params pt1: (x1, y1)
        :params pt2: (x2, y2)
        """
        if value > pt1[0] and value > pt2[0]:
            return 0
        elif value < pt1[0] and value < pt2[0]:
            return 0
        alpha = (pt2[1] - pt1[1]) / (pt2[0] - pt1[0])
        b = pt1[1] - alpha * pt1[0]
        y = value * alpha + b
        y = abs(round(y, 2))
        if 0 <= y <= 1:
            return y
        else:
            return 0

    def function_membership(self, value, trapeze):
        """Belong of a value.
        :params value: value to define the origin
        :params trapeze: the value of the behaviour
        """
        left_bot = (trapeze.x1, 0.0)
        left_top = (trapeze.x2, 1.0)
        right_top = (trapeze.x3, 1.0)
        right_bot = (trapeze.x4, 0.0)
        res1 = self.belong_fall(left_bot, left_top, value)
        res2 = self.belong_line(left_top, right_top, value)
        res3 = self.belong_fall(right_top, right_bot, value)
        val = [x for x in [res1, res2, res3] if 0 < x <= 1]
        try:
            return val[0]
        except:
            return 0

    @timeit
    def main(self):
        """Compute the new cols with values."""
        kid = TRAPEZE(0, 3, 18, 22)
        student = TRAPEZE(18, 22, 26, 28)
        worker = TRAPEZE(26, 28, 55, 60)
        retired = TRAPEZE(55, 60, 95, 120)
        lst_first, lst_second, lst_third, lst_last, lst_id = [], [], [], [], []
        for idx, row in self.df.iterrows():
            age = int(row["Age"])
            lst_first.append(self.function_membership(age, kid))
            lst_second.append(self.function_membership(age, student))
            lst_third.append(self.function_membership(age, worker))
            lst_last.append(self.function_membership(age, retired))
            lst_id.append(row["Id"])
        df_age = pd.DataFrame([lst_first, lst_second, lst_third, lst_last, lst_id]).T
        df_age.columns = ["Kid", "Student", "Worker", "Retired", "Id"]
        return pd.merge(self.df, df_age, on="Id")


df = Fuzzy(df).main()


class TransformCols(object):
    """Transform cols qualitative to quantitative."""

    def __init__(self, df):
        """Constructor."""
        self.df = df

    @timeit
    def main(self):
        """Create for sexe and job for now."""
        self.df["_Sexe"] = self.df["Sexe"].apply(lambda x: 1 if x == "H" else 0)
        self.df["Job"] = Tools.change_nan_value(self.df["Job"], "05")
        self.df["_Job"] = self.df["Job"].apply(lambda x: x[:2])
        self.df["_Job"] = self.df["_Job"].astype(int)
        self.df["Seniority"] = self.df["Seniority"].astype(int)
        self.df["Age"] = self.df["Age"].astype(int)
        return self.df


df = TransformCols(df).main()
df.head()
available_cols = [
    "Age",
    "NewCustomer",
    "Seniority",
    "Premium",
    "CustomerType",
    "CountryId",
    "CustomerActivity",
    "Income",
    "Kid",
    "Student",
    "Worker",
    "Retired",
    "_Sexe",
    "_Job",
]
cols_to_pred = [
    "ind_cco_fin_ult1",
    "ind_recibo_ult1",
    "ind_cder_fin_ult1",
    "ind_cno_fin_ult1",
    "ind_ctju_fin_ult1",
    "ind_ctma_fin_ult1",
    "ind_ctop_fin_ult1",
    "ind_ctpp_fin_ult1",
    "ind_deco_fin_ult1",
    "ind_deme_fin_ult1",
    "ind_dela_fin_ult1",
    "ind_ecue_fin_ult1",
    "ind_fond_fin_ult1",
    "ind_hip_fin_ult1",
    "ind_plan_fin_ult1",
    "ind_pres_fin_ult1",
    "ind_reca_fin_ult1",
    "ind_tjcr_fin_ult1",
    "ind_valo_fin_ult1",
    "ind_viv_fin_ult1",
    "ind_nomina_ult1",
    "ind_nom_pens_ult1",
]
df = df.drop(["Spouse", "PrimaryAdress"], 1)
df_corr = df.corr(method="spearman")
