<kaggle_start><code># # Preperation
import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
import os

for dirname, _, filenames in os.walk("/kaggle/input"):
    for filename in filenames:
        print(os.path.join(dirname, filename))
# Any results you write to the current directory are saved as output.

# # Introduction
# If you are not yet familiar with the basic concepts of pipelines, I recommend you to have a look on following lesson first: https://www.kaggle.com/alexisbcook/pipelines
# Due to performance reasons sklearn transforms returns numpy ndarrays when provided with Dataframes. Unfortunatly like this the column and index information is lost. For some operations, like the OneHotEncoder, this has the risk of mixing up columns. During set up of the preprocessor it would be handy to return pandas dataframes. After this is finished you could switch to the "standard" performant numpy ndarrays.
# This brings the best of both worlds:
# - All nice featurers of dataframes during development
# - Performant sklearn transformes during fitting
# - The ability to easily adapt preprocessing steps and revalidate those.
# This notebook is inspired by lucabasas Notebook on pipelines.
# https://www.kaggle.com/lucabasa/understand-and-use-a-pipeline
# and following article on medium:
# https://medium.com/bigdatarepublic/integrating-pandas-and-scikit-learn-with-pipelines-f70eb6183696
# Herein I suggest further "dataframe transformers" and apply those on the titanic machine learning dataset.
# With this dataframe transformers I developed a full preprocessing pipeline without loosing the dataframe informiation (column and index)
# ## Required packages
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import numpy as np

# ## Load data as Dataframe
# It is important that the test and train data is not concatenated to a complete set and preprocesses together for followig reason:
# For validation a reduced dataset of the train data is used. If we fill emplty values in the train data using information of the test data. We introduced data to the validation that is actually not part of it. Pipelines allow us to develop a preprocessor which can be applied to all datasets seperatly. In turn the validation result will be bether than the exctual prediction.****
# For this reason, only the train data is loaded for now.
df_train = pd.read_csv("../input/titanic/train.csv")
df_train.head()

# ## Take care if nans (imputation)
# create copy for further processing
df = df_train.copy()
df.isnull().sum()

# ## Try and error sklearn transformers with Dataframes
# To keep the dataframe structure we can simply fill the dataframe columns with the ndarray returned by the sklearn transformers.
from sklearn.impute import SimpleImputer

imputer = SimpleImputer()
df[["Age"]] = imputer.fit_transform(df[["Age"]])
df.isnull().sum()


# Alternatively we can create a modified Imputer, which recreates a the original Dataframe from the ndarray after imputation.
class df_SimpleImputer(SimpleImputer):  # (BaseEstimator, TransformerMixin):
    """
    Just a wrapper for the SimpleImputer that keeps the dataframe structure
    """

    def transform(self, X, **kwargs):
        Array = SimpleImputer.transform(self, X, **kwargs)
        return pd.DataFrame(Array, index=X.index, columns=X.columns)


imputer = df_SimpleImputer()
imputer.fit_transform(df[["Age"]]).transpose()


# ## Use ColumnTransformer
# The ColumnTransformer is ideal to process the individual data columns seperatly.However we use the ColumnTransformer, we cannot simply passthrough unprocessed columns and reload the result in the previous dataframe, as the chronology of the ndarray is different from the dataframe. This is shown in following example.
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer

preprocessor = ColumnTransformer(
    transformers=[("num", df_SimpleImputer(), ["Age"])], remainder="passthrough"
)
df = df_train.copy()
array = preprocessor.fit_transform(df)
moddf = pd.DataFrame(array, index=df.index, columns=df.columns)
moddf.head()
# Note!: Columns are incorrectly assigned, chronology of column needs to be modified!

# Using the sklearn transformer can be complicated as the column names and indexes are lost. The representation of the data lacks sufficient overview:
preprocessor = ColumnTransformer(
    transformers=[("num", df_SimpleImputer(), ["Age"])], remainder="passthrough"
)
df = df_train.copy()
nparray = preprocessor.fit_transform(df)
nparray


# ## Modified ColumnTransformer for dataframes
# To solve this problem I created a modified ColumnsTransformer which returns dataframes instead of numpy ndarrays.
class df_ColumnTransformer(ColumnTransformer):
    """
    I cannot garanty that this Transformer works well in combination all sklearn Transformers
    Tested with: SimpleImputer, OneHotEncoder, OrdinalEncoder
    Applies transformers to columns of an array or pandas DataFrame.
    In contrast to the sklearn ColumnTransfromer, this transfomer returns a pandas Dataframe
    This estimator allows different columns or column subsets of the input
    to be transformed separately and the features generated by each transformer
    will be concatenated to form a single feature space.
    This is useful for heterogeneous or columnar data, to combine several
    feature extraction mechanisms or transformations into a single transformer.
    """

    def fit_transform(self, X, *args, **kwargs):
        ColumnTransformer.fit_transform(self, X)
        return df_ColumnTransformer.transform(self, X, *args, **kwargs)

    def transform(self, X, *args, **kwargs):
        ret_df = pd.DataFrame(index=X.index)  # columns=cols)
        for (
            row
        ) in (
            self.transformers_
        ):  # row includes all information for one transformer step.
            transformer = row[1]  # tranformer to be applied
            lst_of_cols_transformed = row[
                2
            ]  # columns to be transformed with this transformer
            if isinstance(transformer, str):
                # passthrough other elments if passthrough is set
                if transformer == "passthrough":
                    cols = []
                    for index in lst_of_cols_transformed:
                        cols.append(X.columns[index])
                    ret_df[cols] = X[cols]
            else:
                transformed = transformer.transform(+X[lst_of_cols_transformed])
                if transformed.shape[1] == len(lst_of_cols_transformed):
                    # Set provided DataFrame columns as given in the cols
                    for i, col in enumerate(lst_of_cols_transformed):
                        if isinstance(transformed, pd.DataFrame):
                            transformed = transformed.values
                        ret_df[col] = transformed[:, i]
                else:
                    # e.g. with OneHotEncoder, more columns are provided, in this case it is not always possible to find
                    # out which output column is related to which input column
                    if isinstance(transformed, pd.DataFrame):
                        ret_df = pd.concat([ret_df, transformed], axis=1)
                    else:
                        cols = []
                        for i in range(1, transformed.shape[1] + 1):
                            cols.append(
                                "-".join(lst_of_cols_transformed) + "_" + str(i)
                            )
                        print(type(transformed))
                        # if tranformer returns ndarray convert to Dataframe
                        if isinstance(transformed, np.ndarray):
                            temp_df = pd.DataFrame(
                                transformed, index=X.index, columns=cols
                            )
                        else:
                            # if sparse matrix is returned by transformer
                            temp_df = pd.DataFrame.sparse.from_spmatrix(
                                transformed, index=X.index, columns=cols
                            )
                            temp_df = temp_df.sparse.to_dense()
                        ret_df = pd.concat([ret_df, temp_df], axis=1)
        return ret_df


# Now we can test it. Here we only need to rename the ColumnTransformer to df_ColumnTransformer. The chronology of the column is according to the transfomer steps, but now the assignment is correct:
#
# Preprocessor #1
preprocessor = df_ColumnTransformer(
    transformers=[("num", df_SimpleImputer(), ["Age"])], remainder="passthrough"
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head(3)

# Checking for nans the combination of both seems to impute all nans correctly.
moddf.isnull().sum()

# ### Further preprocessing steps
# Now we have a ColumnTransfromer for DataFrames, which allows us to work on the preprocessing column by column.
# To present that, we additionally use a different Imputer and an OrdinalEncoder to process the 'Embarked' column.
from sklearn.preprocessing import OrdinalEncoder

# Preprocessor #2
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OrdinalEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head()

# Now we can use dataframe operations to inspect hte modified columns. e.g. We can get the average age per 'Pclass'.
df[["Age", "Pclass"]].groupby("Pclass").mean()

# We can als use the OneHotEncoder instead of the OrdinalEncoder to encode the 'Embarked' column. As the ColumnTransformer does not know which labels were used by the OneHotEncoder, it simply names those Embarked_1, Embarked_2 and Embarked_3.
from sklearn.preprocessing import OneHotEncoder

# Preprocessor #3
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head()


# ## df_OneHotEncoder
# To solve this problem we can write write our own OneHotEncoder, which builds up on top of the original one.
#
class df_OneHotEncoder(OneHotEncoder):
    def transform(self, X, *args, **kwargs):
        uniquedict = {}
        for col in X.columns:
            lst = list(X[col].unique())
            lst.sort()
            uniquedict[col] = lst
        Array = OneHotEncoder.transform(self, X, *args, **kwargs)
        newcolumns = []
        for key in uniquedict:
            for value in uniquedict[key]:
                newcolumns.append(str(key) + "_" + str(value))
        temp_df = pd.DataFrame.sparse.from_spmatrix(
            Array, columns=newcolumns, index=X.index
        )
        temp_df = temp_df.sparse.to_dense()
        return temp_df


# This includes the lables in the columns (Embarked_C, Embarked_Q and Embarked_S).
# Preprocessor #4
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head()

# The df_ColumnTransformer is especially handy if we create our own transformes. In this example we generate one for the Cabin column.
# From that we want to generate a column called 'Cabine_Deck' which extractes the Deck from the cabine numner and a Columns 'Cabine' that indicates if a person had a cabin or not.
# Here is a guideline, which shows how to write your own tranformers/estimators: https://scikit-learn.org/stable/developers/develop.html
from sklearn.base import BaseEstimator, TransformerMixin


class df_Process_Cabin(BaseEstimator, TransformerMixin):
    """
    If a value is given in cell it is set to 1, if nan cell is set to 0
    Empty cabine fields are assigned with 'None'
    """

    def fit(self, *args, **kwargs):
        return self

    def transform(self, X):
        for col in X.columns:
            X[col + "_Deck"] = X[col].str.extract("([A-Za-z]).", expand=False)
            X[col + "_Deck"] = X[col + "_Deck"].fillna("None")
            X[col] = X[col].notnull().astype("int")
        return X


# Just to have this without df as well
class Process_Cabin(df_Process_Cabin):
    # no changes the sklearn transformes will convert this to numpy arrays anyway
    def fit(self, *args, **kwargs):
        return self


# We can use that transformer in the ColumnTransform just as we would use an ordinary sklearn transformers. See column 'Cabin' and 'Cabin_Deck'.
# Preprocessor #5
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", df_Process_Cabin(), ["Cabin"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head(6)

# Now the the new feature 'Cabin_Deck' can be easily analized using the Dataframe functionality. There seems to be some relation between the deck and the Survial rate.
sns.barplot(x="Cabin_Deck", y="Survived", data=moddf)

# As you can see from the dataframe above, the newly genereated column 'Cabin_Deck' still needs to be processed with a OrdinalEncoder or a OneHotEncoder. I coosed the OrdinalEncoder and introduce a Pipeline and another ColumnTransformer for this procedure.
# Preprocessor #6
process_cabin = Pipeline(
    steps=[
        ("cab", df_Process_Cabin()),
        (
            "label_encoder",
            df_ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head(6)


# ## Extract Title from name
# As a next stepy I extract the title from the name. To do that, I generate a specific transformer.
class ExtractTitle(BaseEstimator, TransformerMixin):
    """
    Extracts title from name in titanic machine learning competion
    Basic extraction reused from:
    https://www.kaggle.com/muhammetcimci/easy-titanic-survival-prediction-notebook
    """

    def fit(*args, **kwargs):
        # Nothing to do
        return

    def fit_transform(self, X, *args):
        # Just transform
        return self.transform(X)

    def transform(self, X):
        # Create a list of series from input data.
        # This is needed, as the ColumnTransformer will provide a numpy array
        # I included several options for flexibility, Normally one would be sufficient.
        list_of_Series = []
        if isinstance(X, pd.core.series.Series):
            list_of_Series = [X]
        elif isinstance(X, np.ndarray):
            for i in range(0, X.shape[1]):
                list_of_Series.append(pd.Series(X[:, i]))
        elif isinstance(X, pd.core.frame.DataFrame):
            for col in X.columns:
                list_of_Series.append(X[col])
        else:
            warnings.warn('Datatype "' + str(type(X)) + '" not suppoted!')
        # Create empty numpy array, which holds result later.
        result = np.empty(
            shape=(len(list_of_Series[0]), len(list_of_Series)), dtype="O"
        )
        for i, series in enumerate(list_of_Series):
            title = series.str.extract(" ([A-Za-z]+)\.", expand=False)
            title = title.replace(
                [
                    "Lady",
                    "Capt",
                    "Col",
                    "Don",
                    "Dr",
                    "Major",
                    "Rev",
                    "Jonkheer",
                    "Dona",
                ],
                "Rare",
            )
            title = title.replace(["Countess", "Lady", "Sir"], "Royal")
            title = title.replace("Mlle", "Miss")
            title = title.replace("Ms", "Miss")
            title = title.replace("Mme", "Mrs")
            result[:, i] = title.values
        return result


#
class df_ExtractTitle(ExtractTitle):
    """
    Extracts title from name in titanic machine learning competion
    Returns Dataframe
    """

    def transform(self, X):
        columns = list(X.columns)
        for i, col in enumerate(columns):
            columns[i] = col + "_Title"
        matrix = ExtractTitle.transform(self, X)
        return pd.DataFrame(matrix, index=X.index, columns=columns)


# Lets apply the newly created tranformer. Further I include an OrdinalEncoder for the column 'Sex'. I guess you can imagine how complicated it would be to evaluate the correct behavior using the numpy arrays.
# Preprocessor #7
process_cabin = Pipeline(
    steps=[
        ("cab", df_Process_Cabin()),
        (
            "label_encoder",
            df_ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
process_title = Pipeline(
    steps=[("title_extract", df_ExtractTitle()), ("encode", df_OneHotEncoder())]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
        ("name", process_title, ["Name"]),
        ("sex", OrdinalEncoder(), ["Sex"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head(4)


# The column 'Ticket' is not needed in my prediction. Also the column 'Survived' should not be processed. Instead of passing all unprocessed columns (remainder = passthrough), I will copy unchanged columns. To do so I created a transformer copier. For sure I could as well just use an Imputer, which has nothing to do for those columns. For the column 'Pclass' I will include a OneHotEncoder, later I will test if that provides better results during validation. this is the last preprocessing step I will apply.
class Copier(BaseEstimator, TransformerMixin):
    """
    Simply takes over column as they are.
    """

    def fit(*args, **kwargs):
        # Nothing to do
        return

    def fit_transform(self, X, *args):
        # Just transform
        return self.transform(X)

    def transform(self, X):
        return X


# Here you can see the complete preprocessing Pipeline using the Dataframes.
# Preprocessor #8
process_cabin = Pipeline(
    steps=[
        ("cab", df_Process_Cabin()),
        (
            "label_encoder",
            df_ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
process_title = Pipeline(
    steps=[("title_extract", df_ExtractTitle()), ("encode", df_OneHotEncoder())]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
        ("name", process_title, ["Name"]),
        ("sex", OrdinalEncoder(), ["Sex"]),
        ("onehot", df_OneHotEncoder(), ["Pclass"]),
        ("unmodified", Copier(), ["Fare", "Parch", "SibSp"]),
    ],
    remainder="drop",
)  # drops ticket
df = df_train.copy()

# I will make a time measurment on the compete preprocessing.
moddf = preprocessor.fit_transform(df)
moddf.head(3)

# ## Switch back to the performant numpy arrays
# Now that the preprocessing is successfully developed, the pipeline can be modified to work with the perfromant numpy arrays. To do that, from the previous pipeline I removed all 'df_'s, switching to the ordinary sklearn transformers.
# Sklearn Preprocessor
process_cabin = Pipeline(
    steps=[
        ("cab", Process_Cabin()),
        (
            "label_encoder",
            ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder()),
    ]
)
process_title = Pipeline(
    steps=[("title_extract", ExtractTitle()), ("encode", OneHotEncoder())]
)
preprocessor = ColumnTransformer(
    transformers=[
        ("num", SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
        ("name", process_title, ["Name"]),
        ("sex", OrdinalEncoder(), ["Sex"]),
        ("onehot", OneHotEncoder(), ["Pclass"]),
        ("unmodified", Copier(), ["Fare", "Parch", "SibSp"]),
    ],
    remainder="drop",
)  # drops ticket
df = df_train.copy()
nparray = preprocessor.fit_transform(df)
nparray[1, :]

# Compared to our previous preprocessing pipeline we were significantly faster. From ~125 ms down to ~44.8 ms.
# We can also show that the df_ version and the numpy array version returned the same result. From now on we will not need the Dataframe version anymore.
if np.sum(np.equal(nparray, moddf.values)) == nparray.size:
    print("All elements are equal!")

# # Create Pipeline with Model
# Now that preprocessor is finished we can create a complete pipeline to include our model as well.
# Preproccesing-Modeling Pipeline #1
from sklearn.ensemble import GradientBoostingClassifier

process_cabin = Pipeline(
    steps=[
        ("cab", Process_Cabin()),
        (
            "label_encoder",
            ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder(handle_unknown="ignore")),
    ]
)
process_title = Pipeline(
    steps=[("title_extract", ExtractTitle()), ("encode", OneHotEncoder())]
)
preprocessor = ColumnTransformer(
    transformers=[
        ("num", SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
        ("name", process_title, ["Name"]),
        ("sex", OrdinalEncoder(), ["Sex"]),
        ("onehot", OneHotEncoder(handle_unknown="ignore"), ["Pclass"]),
        ("unmodified", Copier(), ["Fare", "Parch", "SibSp"]),
    ],
    remainder="drop",
)  # drops ticket
model = GradientBoostingClassifier(random_state=0)
process = Pipeline(steps=[("preprocessor", preprocessor), ("model", model)])
from sklearn.model_selection import cross_val_score

df = df_train.copy()
y = df["Survived"]
df = df.drop(
    "Survived", axis=1
)  # make sure that Survived is really not used in fitting
X = df
cross_val_score(process, X, y).mean()

# Now we can play around with the Pipeline and see which features and which models perform best in cross validation.
# Preproccesing-Modeling Pipeline #2
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split

process_cabin = Pipeline(
    steps=[
        ("cab", Process_Cabin()),
        (
            "label_encoder",
            ColumnTransformer(
                transformers=[
                    (
                        "deck",
                        OneHotEncoder(handle_unknown="ignore", categories="auto"),
                        ["Cabin_Deck"],
                    ),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder(handle_unknown="ignore", categories="auto")),
    ]
)
process_title = Pipeline(
    steps=[
        ("title_extract", ExtractTitle()),
        ("encode", OneHotEncoder(handle_unknown="ignore", categories="auto")),
    ]
)
preprocessor = ColumnTransformer(
    transformers=[
        ("num", SimpleImputer(), ["Age", "Fare"]),  # Fare has nan in the test data
        ("cab", cabine, ["Cabin"]),  # Categorize and imputes at the same time
        ("cat", OrdinalEncoder(), ["Sex"]),
        ("class", OrdinalEncoder(), ["Pclass"]),
        ("parch", impute_onehot, ["Parch"]),
        ("embarked", impute_onehot, ["Embarked"]),
        ("name", title_onehot, ["Name"]),
        ("copy", Copier(), ["SibSp"]),
    ],
    remainder="drop",
)  # drops ticket
# Validate
df = df_train.copy()
y = df["Survived"].astype(int).values
df = df.drop(
    "Survived", axis=1
)  # make sure that Survived is really not used in fitting
X = preprocessor.fit_transform(df)
# Validation set for stopping round of XGBclassifier
__, val_x, __, val_y = train_test_split(X, y, test_size=0.30, random_state=4)
model = XGBClassifier(
    early_stopping_round=5, eval_set=(val_x, val_y), learning_rate=0.08, random_state=8
)
cross_val_score(model, X, y, cv=4).mean()

# At the end we can use our final pipeline to predict the Survived feature in the test data.
# train
df = df_train.copy()
y_train = df_train["Survived"].values
df = df.drop(["Survived"], axis=1)
X_train = df
X_test = pd.read_csv("../input/titanic/test.csv")
# Validation set for stopping round of XGBclassifier -> Complete X_train and y_train can be used.
X_train_class = preprocessor.fit_transform(
    X_train
)  # preprocessor should be performed on this data as well.
model = XGBClassifier(
    early_stopping_round=8,
    eval_set=(X_train_class, y_train),
    learning_rate=0.08,
    random_state=8,
)
process = Pipeline(steps=[("preprocessor", preprocessor), ("model", model)])
process.fit(X_train, y_train)
y_pred = process.predict(X_test)
y_pred
output = pd.DataFrame({"PassengerId": X_test["PassengerId"].values, "Survived": y_pred})
output
output.to_csv("submission.csv", index=False)


def auto_model_cross_val(
    X,
    y,
    preprocessor,
    model_class_ptr,
    optimize_param,
    param_list,
    model_kwargs=dict(),
    cross_val_kwargs={"scoring": "accuracy", "cv": 5},
):
    """
    Preform multiple cross_validations of model with modified model parameter
    :param val_set:
    :param y_name:
    :param model_class_ptr:
    :param optimize_param:
    :param param_list:
    :param model_kwargs:
    :return:
    """
    from sklearn.model_selection import cross_val_score
    from sklearn.pipeline import Pipeline
    import pandas as pd

    score_df = pd.DataFrame()
    score_df[optimize_param] = param_list
    score_df = score_df.set_index(optimize_param)
    score_list = []
    for param in param_list:
        model_kwargs[optimize_param] = param
        model_instance = model_class_ptr(**model_kwargs)
        process = Pipeline(
            steps=[("preprocessor", preprocessor), ("model", model_instance)]
        )
        score = cross_val_score(process, X, y, **cross_val_kwargs).mean()
        score_list.append(score)
    score_df[cross_val_kwargs["scoring"]] = score_list
    return score_df


from xgboost import XGBClassifier

y_train = df_train["Survived"].astype(int).values
impute_onehot = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder(handle_unknown="ignore", categories="auto")),
    ]
)
title_onehot = Pipeline(
    steps=[
        ("title_extract", ExtractTitle()),
        ("one_hot", OneHotEncoder(handle_unknown="ignore", categories="auto")),
    ]
)
cabine = Pipeline(
    steps=[
        ("cab", Process_Cabin()),
        (
            "label_encoder",
            ColumnTransformer(
                transformers=[
                    ("deck", OneHotEncoder(handle_unknown="ignore"), ["Cabin_Deck"])
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
preprocessor = ColumnTransformer(
    transformers=[
        ("num", SimpleImputer(), ["Age", "Fare"]),  # Fare has nan in the test data
        ("cab", cabine, ["Cabin"]),  # Categorize and imputes at the same time
        ("cat", OrdinalEncoder(), ["Sex"]),
        ("class", OrdinalEncoder(), ["Pclass"]),
        ("parch", impute_onehot, ["Parch"]),
        ("embarked", impute_onehot, ["Embarked"]),
        ("name", title_onehot, ["Name"]),
        ("copy", Copier(), ["SibSp"]),
    ]
)
__, val_x, __, val_y = train_test_split(
    df_train, y_train, test_size=0.30, random_state=8
)
auto_model_cross_val(
    df_train,
    y_train,
    preprocessor,
    XGBClassifier,
    "learning_rate",
    param_list=[0.06, 0.07, 0.08, 0.09, 0.1],
    model_kwargs={"early_stopping_rounds": 8, "eval_set": [(val_x, val_y)]},
)
<|endoftext|>
[{"Id": 32224913, "ScriptId": 8825052, "ParentScriptVersionId": NaN, "ScriptLanguageId": 9, "AuthorUserId": 4714017, "CreationDate": "04/18/2020 09:04:12", "VersionNumber": 7.0, "Title": "Step by step Pipeline with pandas Dataframes", "EvaluationDate": "04/18/2020", "IsChange": true, "TotalLines": 751.0, "LinesInsertedFromPrevious": 0.0, "LinesChangedFromPrevious": 0.0, "LinesUnchangedFromPrevious": 751.0, "LinesInsertedFromFork": NaN, "LinesDeletedFromFork": NaN, "LinesChangedFromFork": NaN, "LinesUnchangedFromFork": NaN, "TotalVotes": 0}]
# # Preperation
import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
import os

for dirname, _, filenames in os.walk("/kaggle/input"):
    for filename in filenames:
        print(os.path.join(dirname, filename))
# Any results you write to the current directory are saved as output.

# # Introduction
# If you are not yet familiar with the basic concepts of pipelines, I recommend you to have a look on following lesson first: https://www.kaggle.com/alexisbcook/pipelines
# Due to performance reasons sklearn transforms returns numpy ndarrays when provided with Dataframes. Unfortunatly like this the column and index information is lost. For some operations, like the OneHotEncoder, this has the risk of mixing up columns. During set up of the preprocessor it would be handy to return pandas dataframes. After this is finished you could switch to the "standard" performant numpy ndarrays.
# This brings the best of both worlds:
# - All nice featurers of dataframes during development
# - Performant sklearn transformes during fitting
# - The ability to easily adapt preprocessing steps and revalidate those.
# This notebook is inspired by lucabasas Notebook on pipelines.
# https://www.kaggle.com/lucabasa/understand-and-use-a-pipeline
# and following article on medium:
# https://medium.com/bigdatarepublic/integrating-pandas-and-scikit-learn-with-pipelines-f70eb6183696
# Herein I suggest further "dataframe transformers" and apply those on the titanic machine learning dataset.
# With this dataframe transformers I developed a full preprocessing pipeline without loosing the dataframe informiation (column and index)
# ## Required packages
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import numpy as np

# ## Load data as Dataframe
# It is important that the test and train data is not concatenated to a complete set and preprocesses together for followig reason:
# For validation a reduced dataset of the train data is used. If we fill emplty values in the train data using information of the test data. We introduced data to the validation that is actually not part of it. Pipelines allow us to develop a preprocessor which can be applied to all datasets seperatly. In turn the validation result will be bether than the exctual prediction.****
# For this reason, only the train data is loaded for now.
df_train = pd.read_csv("../input/titanic/train.csv")
df_train.head()

# ## Take care if nans (imputation)
# create copy for further processing
df = df_train.copy()
df.isnull().sum()

# ## Try and error sklearn transformers with Dataframes
# To keep the dataframe structure we can simply fill the dataframe columns with the ndarray returned by the sklearn transformers.
from sklearn.impute import SimpleImputer

imputer = SimpleImputer()
df[["Age"]] = imputer.fit_transform(df[["Age"]])
df.isnull().sum()


# Alternatively we can create a modified Imputer, which recreates a the original Dataframe from the ndarray after imputation.
class df_SimpleImputer(SimpleImputer):  # (BaseEstimator, TransformerMixin):
    """
    Just a wrapper for the SimpleImputer that keeps the dataframe structure
    """

    def transform(self, X, **kwargs):
        Array = SimpleImputer.transform(self, X, **kwargs)
        return pd.DataFrame(Array, index=X.index, columns=X.columns)


imputer = df_SimpleImputer()
imputer.fit_transform(df[["Age"]]).transpose()


# ## Use ColumnTransformer
# The ColumnTransformer is ideal to process the individual data columns seperatly.However we use the ColumnTransformer, we cannot simply passthrough unprocessed columns and reload the result in the previous dataframe, as the chronology of the ndarray is different from the dataframe. This is shown in following example.
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.compose import ColumnTransformer

preprocessor = ColumnTransformer(
    transformers=[("num", df_SimpleImputer(), ["Age"])], remainder="passthrough"
)
df = df_train.copy()
array = preprocessor.fit_transform(df)
moddf = pd.DataFrame(array, index=df.index, columns=df.columns)
moddf.head()
# Note!: Columns are incorrectly assigned, chronology of column needs to be modified!

# Using the sklearn transformer can be complicated as the column names and indexes are lost. The representation of the data lacks sufficient overview:
preprocessor = ColumnTransformer(
    transformers=[("num", df_SimpleImputer(), ["Age"])], remainder="passthrough"
)
df = df_train.copy()
nparray = preprocessor.fit_transform(df)
nparray


# ## Modified ColumnTransformer for dataframes
# To solve this problem I created a modified ColumnsTransformer which returns dataframes instead of numpy ndarrays.
class df_ColumnTransformer(ColumnTransformer):
    """
    I cannot garanty that this Transformer works well in combination all sklearn Transformers
    Tested with: SimpleImputer, OneHotEncoder, OrdinalEncoder
    Applies transformers to columns of an array or pandas DataFrame.
    In contrast to the sklearn ColumnTransfromer, this transfomer returns a pandas Dataframe
    This estimator allows different columns or column subsets of the input
    to be transformed separately and the features generated by each transformer
    will be concatenated to form a single feature space.
    This is useful for heterogeneous or columnar data, to combine several
    feature extraction mechanisms or transformations into a single transformer.
    """

    def fit_transform(self, X, *args, **kwargs):
        ColumnTransformer.fit_transform(self, X)
        return df_ColumnTransformer.transform(self, X, *args, **kwargs)

    def transform(self, X, *args, **kwargs):
        ret_df = pd.DataFrame(index=X.index)  # columns=cols)
        for (
            row
        ) in (
            self.transformers_
        ):  # row includes all information for one transformer step.
            transformer = row[1]  # tranformer to be applied
            lst_of_cols_transformed = row[
                2
            ]  # columns to be transformed with this transformer
            if isinstance(transformer, str):
                # passthrough other elments if passthrough is set
                if transformer == "passthrough":
                    cols = []
                    for index in lst_of_cols_transformed:
                        cols.append(X.columns[index])
                    ret_df[cols] = X[cols]
            else:
                transformed = transformer.transform(+X[lst_of_cols_transformed])
                if transformed.shape[1] == len(lst_of_cols_transformed):
                    # Set provided DataFrame columns as given in the cols
                    for i, col in enumerate(lst_of_cols_transformed):
                        if isinstance(transformed, pd.DataFrame):
                            transformed = transformed.values
                        ret_df[col] = transformed[:, i]
                else:
                    # e.g. with OneHotEncoder, more columns are provided, in this case it is not always possible to find
                    # out which output column is related to which input column
                    if isinstance(transformed, pd.DataFrame):
                        ret_df = pd.concat([ret_df, transformed], axis=1)
                    else:
                        cols = []
                        for i in range(1, transformed.shape[1] + 1):
                            cols.append(
                                "-".join(lst_of_cols_transformed) + "_" + str(i)
                            )
                        print(type(transformed))
                        # if tranformer returns ndarray convert to Dataframe
                        if isinstance(transformed, np.ndarray):
                            temp_df = pd.DataFrame(
                                transformed, index=X.index, columns=cols
                            )
                        else:
                            # if sparse matrix is returned by transformer
                            temp_df = pd.DataFrame.sparse.from_spmatrix(
                                transformed, index=X.index, columns=cols
                            )
                            temp_df = temp_df.sparse.to_dense()
                        ret_df = pd.concat([ret_df, temp_df], axis=1)
        return ret_df


# Now we can test it. Here we only need to rename the ColumnTransformer to df_ColumnTransformer. The chronology of the column is according to the transfomer steps, but now the assignment is correct:
#
# Preprocessor #1
preprocessor = df_ColumnTransformer(
    transformers=[("num", df_SimpleImputer(), ["Age"])], remainder="passthrough"
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head(3)

# Checking for nans the combination of both seems to impute all nans correctly.
moddf.isnull().sum()

# ### Further preprocessing steps
# Now we have a ColumnTransfromer for DataFrames, which allows us to work on the preprocessing column by column.
# To present that, we additionally use a different Imputer and an OrdinalEncoder to process the 'Embarked' column.
from sklearn.preprocessing import OrdinalEncoder

# Preprocessor #2
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OrdinalEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head()

# Now we can use dataframe operations to inspect hte modified columns. e.g. We can get the average age per 'Pclass'.
df[["Age", "Pclass"]].groupby("Pclass").mean()

# We can als use the OneHotEncoder instead of the OrdinalEncoder to encode the 'Embarked' column. As the ColumnTransformer does not know which labels were used by the OneHotEncoder, it simply names those Embarked_1, Embarked_2 and Embarked_3.
from sklearn.preprocessing import OneHotEncoder

# Preprocessor #3
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head()


# ## df_OneHotEncoder
# To solve this problem we can write write our own OneHotEncoder, which builds up on top of the original one.
#
class df_OneHotEncoder(OneHotEncoder):
    def transform(self, X, *args, **kwargs):
        uniquedict = {}
        for col in X.columns:
            lst = list(X[col].unique())
            lst.sort()
            uniquedict[col] = lst
        Array = OneHotEncoder.transform(self, X, *args, **kwargs)
        newcolumns = []
        for key in uniquedict:
            for value in uniquedict[key]:
                newcolumns.append(str(key) + "_" + str(value))
        temp_df = pd.DataFrame.sparse.from_spmatrix(
            Array, columns=newcolumns, index=X.index
        )
        temp_df = temp_df.sparse.to_dense()
        return temp_df


# This includes the lables in the columns (Embarked_C, Embarked_Q and Embarked_S).
# Preprocessor #4
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head()

# The df_ColumnTransformer is especially handy if we create our own transformes. In this example we generate one for the Cabin column.
# From that we want to generate a column called 'Cabine_Deck' which extractes the Deck from the cabine numner and a Columns 'Cabine' that indicates if a person had a cabin or not.
# Here is a guideline, which shows how to write your own tranformers/estimators: https://scikit-learn.org/stable/developers/develop.html
from sklearn.base import BaseEstimator, TransformerMixin


class df_Process_Cabin(BaseEstimator, TransformerMixin):
    """
    If a value is given in cell it is set to 1, if nan cell is set to 0
    Empty cabine fields are assigned with 'None'
    """

    def fit(self, *args, **kwargs):
        return self

    def transform(self, X):
        for col in X.columns:
            X[col + "_Deck"] = X[col].str.extract("([A-Za-z]).", expand=False)
            X[col + "_Deck"] = X[col + "_Deck"].fillna("None")
            X[col] = X[col].notnull().astype("int")
        return X


# Just to have this without df as well
class Process_Cabin(df_Process_Cabin):
    # no changes the sklearn transformes will convert this to numpy arrays anyway
    def fit(self, *args, **kwargs):
        return self


# We can use that transformer in the ColumnTransform just as we would use an ordinary sklearn transformers. See column 'Cabin' and 'Cabin_Deck'.
# Preprocessor #5
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", df_Process_Cabin(), ["Cabin"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head(6)

# Now the the new feature 'Cabin_Deck' can be easily analized using the Dataframe functionality. There seems to be some relation between the deck and the Survial rate.
sns.barplot(x="Cabin_Deck", y="Survived", data=moddf)

# As you can see from the dataframe above, the newly genereated column 'Cabin_Deck' still needs to be processed with a OrdinalEncoder or a OneHotEncoder. I coosed the OrdinalEncoder and introduce a Pipeline and another ColumnTransformer for this procedure.
# Preprocessor #6
process_cabin = Pipeline(
    steps=[
        ("cab", df_Process_Cabin()),
        (
            "label_encoder",
            df_ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head(6)


# ## Extract Title from name
# As a next stepy I extract the title from the name. To do that, I generate a specific transformer.
class ExtractTitle(BaseEstimator, TransformerMixin):
    """
    Extracts title from name in titanic machine learning competion
    Basic extraction reused from:
    https://www.kaggle.com/muhammetcimci/easy-titanic-survival-prediction-notebook
    """

    def fit(*args, **kwargs):
        # Nothing to do
        return

    def fit_transform(self, X, *args):
        # Just transform
        return self.transform(X)

    def transform(self, X):
        # Create a list of series from input data.
        # This is needed, as the ColumnTransformer will provide a numpy array
        # I included several options for flexibility, Normally one would be sufficient.
        list_of_Series = []
        if isinstance(X, pd.core.series.Series):
            list_of_Series = [X]
        elif isinstance(X, np.ndarray):
            for i in range(0, X.shape[1]):
                list_of_Series.append(pd.Series(X[:, i]))
        elif isinstance(X, pd.core.frame.DataFrame):
            for col in X.columns:
                list_of_Series.append(X[col])
        else:
            warnings.warn('Datatype "' + str(type(X)) + '" not suppoted!')
        # Create empty numpy array, which holds result later.
        result = np.empty(
            shape=(len(list_of_Series[0]), len(list_of_Series)), dtype="O"
        )
        for i, series in enumerate(list_of_Series):
            title = series.str.extract(" ([A-Za-z]+)\.", expand=False)
            title = title.replace(
                [
                    "Lady",
                    "Capt",
                    "Col",
                    "Don",
                    "Dr",
                    "Major",
                    "Rev",
                    "Jonkheer",
                    "Dona",
                ],
                "Rare",
            )
            title = title.replace(["Countess", "Lady", "Sir"], "Royal")
            title = title.replace("Mlle", "Miss")
            title = title.replace("Ms", "Miss")
            title = title.replace("Mme", "Mrs")
            result[:, i] = title.values
        return result


#
class df_ExtractTitle(ExtractTitle):
    """
    Extracts title from name in titanic machine learning competion
    Returns Dataframe
    """

    def transform(self, X):
        columns = list(X.columns)
        for i, col in enumerate(columns):
            columns[i] = col + "_Title"
        matrix = ExtractTitle.transform(self, X)
        return pd.DataFrame(matrix, index=X.index, columns=columns)


# Lets apply the newly created tranformer. Further I include an OrdinalEncoder for the column 'Sex'. I guess you can imagine how complicated it would be to evaluate the correct behavior using the numpy arrays.
# Preprocessor #7
process_cabin = Pipeline(
    steps=[
        ("cab", df_Process_Cabin()),
        (
            "label_encoder",
            df_ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
process_title = Pipeline(
    steps=[("title_extract", df_ExtractTitle()), ("encode", df_OneHotEncoder())]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
        ("name", process_title, ["Name"]),
        ("sex", OrdinalEncoder(), ["Sex"]),
    ],
    remainder="passthrough",
)
df = df_train.copy()
moddf = preprocessor.fit_transform(df)
moddf.head(4)


# The column 'Ticket' is not needed in my prediction. Also the column 'Survived' should not be processed. Instead of passing all unprocessed columns (remainder = passthrough), I will copy unchanged columns. To do so I created a transformer copier. For sure I could as well just use an Imputer, which has nothing to do for those columns. For the column 'Pclass' I will include a OneHotEncoder, later I will test if that provides better results during validation. this is the last preprocessing step I will apply.
class Copier(BaseEstimator, TransformerMixin):
    """
    Simply takes over column as they are.
    """

    def fit(*args, **kwargs):
        # Nothing to do
        return

    def fit_transform(self, X, *args):
        # Just transform
        return self.transform(X)

    def transform(self, X):
        return X


# Here you can see the complete preprocessing Pipeline using the Dataframes.
# Preprocessor #8
process_cabin = Pipeline(
    steps=[
        ("cab", df_Process_Cabin()),
        (
            "label_encoder",
            df_ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", df_SimpleImputer(strategy="most_frequent")),
        # The df_OneHotEncoder requires a Dataframe as input, therefore df_SimpleImputer must be used instead of the SimpleImputer.
        ("label_encoder", df_OneHotEncoder()),
    ]
)
process_title = Pipeline(
    steps=[("title_extract", df_ExtractTitle()), ("encode", df_OneHotEncoder())]
)
preprocessor = df_ColumnTransformer(
    transformers=[
        ("num", df_SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
        ("name", process_title, ["Name"]),
        ("sex", OrdinalEncoder(), ["Sex"]),
        ("onehot", df_OneHotEncoder(), ["Pclass"]),
        ("unmodified", Copier(), ["Fare", "Parch", "SibSp"]),
    ],
    remainder="drop",
)  # drops ticket
df = df_train.copy()

# I will make a time measurment on the compete preprocessing.
moddf = preprocessor.fit_transform(df)
moddf.head(3)

# ## Switch back to the performant numpy arrays
# Now that the preprocessing is successfully developed, the pipeline can be modified to work with the perfromant numpy arrays. To do that, from the previous pipeline I removed all 'df_'s, switching to the ordinary sklearn transformers.
# Sklearn Preprocessor
process_cabin = Pipeline(
    steps=[
        ("cab", Process_Cabin()),
        (
            "label_encoder",
            ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder()),
    ]
)
process_title = Pipeline(
    steps=[("title_extract", ExtractTitle()), ("encode", OneHotEncoder())]
)
preprocessor = ColumnTransformer(
    transformers=[
        ("num", SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
        ("name", process_title, ["Name"]),
        ("sex", OrdinalEncoder(), ["Sex"]),
        ("onehot", OneHotEncoder(), ["Pclass"]),
        ("unmodified", Copier(), ["Fare", "Parch", "SibSp"]),
    ],
    remainder="drop",
)  # drops ticket
df = df_train.copy()
nparray = preprocessor.fit_transform(df)
nparray[1, :]

# Compared to our previous preprocessing pipeline we were significantly faster. From ~125 ms down to ~44.8 ms.
# We can also show that the df_ version and the numpy array version returned the same result. From now on we will not need the Dataframe version anymore.
if np.sum(np.equal(nparray, moddf.values)) == nparray.size:
    print("All elements are equal!")

# # Create Pipeline with Model
# Now that preprocessor is finished we can create a complete pipeline to include our model as well.
# Preproccesing-Modeling Pipeline #1
from sklearn.ensemble import GradientBoostingClassifier

process_cabin = Pipeline(
    steps=[
        ("cab", Process_Cabin()),
        (
            "label_encoder",
            ColumnTransformer(
                transformers=[
                    ("deck", OrdinalEncoder(), ["Cabin_Deck"]),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder(handle_unknown="ignore")),
    ]
)
process_title = Pipeline(
    steps=[("title_extract", ExtractTitle()), ("encode", OneHotEncoder())]
)
preprocessor = ColumnTransformer(
    transformers=[
        ("num", SimpleImputer(), ["Age"]),
        ("embarked", process_embarked, ["Embarked"]),
        ("cabin", process_cabin, ["Cabin"]),
        ("name", process_title, ["Name"]),
        ("sex", OrdinalEncoder(), ["Sex"]),
        ("onehot", OneHotEncoder(handle_unknown="ignore"), ["Pclass"]),
        ("unmodified", Copier(), ["Fare", "Parch", "SibSp"]),
    ],
    remainder="drop",
)  # drops ticket
model = GradientBoostingClassifier(random_state=0)
process = Pipeline(steps=[("preprocessor", preprocessor), ("model", model)])
from sklearn.model_selection import cross_val_score

df = df_train.copy()
y = df["Survived"]
df = df.drop(
    "Survived", axis=1
)  # make sure that Survived is really not used in fitting
X = df
cross_val_score(process, X, y).mean()

# Now we can play around with the Pipeline and see which features and which models perform best in cross validation.
# Preproccesing-Modeling Pipeline #2
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split

process_cabin = Pipeline(
    steps=[
        ("cab", Process_Cabin()),
        (
            "label_encoder",
            ColumnTransformer(
                transformers=[
                    (
                        "deck",
                        OneHotEncoder(handle_unknown="ignore", categories="auto"),
                        ["Cabin_Deck"],
                    ),
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
process_embarked = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder(handle_unknown="ignore", categories="auto")),
    ]
)
process_title = Pipeline(
    steps=[
        ("title_extract", ExtractTitle()),
        ("encode", OneHotEncoder(handle_unknown="ignore", categories="auto")),
    ]
)
preprocessor = ColumnTransformer(
    transformers=[
        ("num", SimpleImputer(), ["Age", "Fare"]),  # Fare has nan in the test data
        ("cab", cabine, ["Cabin"]),  # Categorize and imputes at the same time
        ("cat", OrdinalEncoder(), ["Sex"]),
        ("class", OrdinalEncoder(), ["Pclass"]),
        ("parch", impute_onehot, ["Parch"]),
        ("embarked", impute_onehot, ["Embarked"]),
        ("name", title_onehot, ["Name"]),
        ("copy", Copier(), ["SibSp"]),
    ],
    remainder="drop",
)  # drops ticket
# Validate
df = df_train.copy()
y = df["Survived"].astype(int).values
df = df.drop(
    "Survived", axis=1
)  # make sure that Survived is really not used in fitting
X = preprocessor.fit_transform(df)
# Validation set for stopping round of XGBclassifier
__, val_x, __, val_y = train_test_split(X, y, test_size=0.30, random_state=4)
model = XGBClassifier(
    early_stopping_round=5, eval_set=(val_x, val_y), learning_rate=0.08, random_state=8
)
cross_val_score(model, X, y, cv=4).mean()

# At the end we can use our final pipeline to predict the Survived feature in the test data.
# train
df = df_train.copy()
y_train = df_train["Survived"].values
df = df.drop(["Survived"], axis=1)
X_train = df
X_test = pd.read_csv("../input/titanic/test.csv")
# Validation set for stopping round of XGBclassifier -> Complete X_train and y_train can be used.
X_train_class = preprocessor.fit_transform(
    X_train
)  # preprocessor should be performed on this data as well.
model = XGBClassifier(
    early_stopping_round=8,
    eval_set=(X_train_class, y_train),
    learning_rate=0.08,
    random_state=8,
)
process = Pipeline(steps=[("preprocessor", preprocessor), ("model", model)])
process.fit(X_train, y_train)
y_pred = process.predict(X_test)
y_pred
output = pd.DataFrame({"PassengerId": X_test["PassengerId"].values, "Survived": y_pred})
output
output.to_csv("submission.csv", index=False)


def auto_model_cross_val(
    X,
    y,
    preprocessor,
    model_class_ptr,
    optimize_param,
    param_list,
    model_kwargs=dict(),
    cross_val_kwargs={"scoring": "accuracy", "cv": 5},
):
    """
    Preform multiple cross_validations of model with modified model parameter
    :param val_set:
    :param y_name:
    :param model_class_ptr:
    :param optimize_param:
    :param param_list:
    :param model_kwargs:
    :return:
    """
    from sklearn.model_selection import cross_val_score
    from sklearn.pipeline import Pipeline
    import pandas as pd

    score_df = pd.DataFrame()
    score_df[optimize_param] = param_list
    score_df = score_df.set_index(optimize_param)
    score_list = []
    for param in param_list:
        model_kwargs[optimize_param] = param
        model_instance = model_class_ptr(**model_kwargs)
        process = Pipeline(
            steps=[("preprocessor", preprocessor), ("model", model_instance)]
        )
        score = cross_val_score(process, X, y, **cross_val_kwargs).mean()
        score_list.append(score)
    score_df[cross_val_kwargs["scoring"]] = score_list
    return score_df


from xgboost import XGBClassifier

y_train = df_train["Survived"].astype(int).values
impute_onehot = Pipeline(
    steps=[
        ("imputer", SimpleImputer(strategy="most_frequent")),
        ("label_encoder", OneHotEncoder(handle_unknown="ignore", categories="auto")),
    ]
)
title_onehot = Pipeline(
    steps=[
        ("title_extract", ExtractTitle()),
        ("one_hot", OneHotEncoder(handle_unknown="ignore", categories="auto")),
    ]
)
cabine = Pipeline(
    steps=[
        ("cab", Process_Cabin()),
        (
            "label_encoder",
            ColumnTransformer(
                transformers=[
                    ("deck", OneHotEncoder(handle_unknown="ignore"), ["Cabin_Deck"])
                ],
                remainder="passthrough",
            ),
        ),
    ]
)
preprocessor = ColumnTransformer(
    transformers=[
        ("num", SimpleImputer(), ["Age", "Fare"]),  # Fare has nan in the test data
        ("cab", cabine, ["Cabin"]),  # Categorize and imputes at the same time
        ("cat", OrdinalEncoder(), ["Sex"]),
        ("class", OrdinalEncoder(), ["Pclass"]),
        ("parch", impute_onehot, ["Parch"]),
        ("embarked", impute_onehot, ["Embarked"]),
        ("name", title_onehot, ["Name"]),
        ("copy", Copier(), ["SibSp"]),
    ]
)
__, val_x, __, val_y = train_test_split(
    df_train, y_train, test_size=0.30, random_state=8
)
auto_model_cross_val(
    df_train,
    y_train,
    preprocessor,
    XGBClassifier,
    "learning_rate",
    param_list=[0.06, 0.07, 0.08, 0.09, 0.1],
    model_kwargs={"early_stopping_rounds": 8, "eval_set": [(val_x, val_y)]},
)
