<kaggle_start><code>import os
import warnings
from tqdm import tqdm
import pandas as pd
import numpy as np
from ml_stratifiers import MultilabelStratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.base import clone
from sklearn.linear_model import LogisticRegression
from sklearn.multioutput import MultiOutputClassifier
from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA

warnings.filterwarnings("ignore")


def load_datasets(data_path):
    train_features = pd.read_csv(os.path.join(data_path, "train_features.csv"))
    train_targets_s = pd.read_csv(os.path.join(data_path, "train_targets_scored.csv"))
    train_targets_n = pd.read_csv(
        os.path.join(data_path, "train_targets_nonscored.csv")
    )
    test_features = pd.read_csv(os.path.join(data_path, "test_features.csv"))
    sample_submission = pd.read_csv(os.path.join(data_path, "sample_submission.csv"))
    train_features.set_index("sig_id", inplace=True)
    train_targets_s.set_index("sig_id", inplace=True)
    test_features.set_index("sig_id", inplace=True)
    sample_submission.set_index("sig_id", inplace=True)
    assert np.all(test_features.index == sample_submission.index)
    return (
        train_features,
        train_targets_s,
        train_targets_n,
        test_features,
        sample_submission,
    )


def preprocess(X, y, test, random_state=2020):
    X, y, test = X.copy(), y.copy(), test.copy()
    # 1. Filter cp_type=ctl_vehicle
    X = X[X.cp_type != "ctl_vehicle"]
    X.drop("cp_type", axis=1, inplace=True)
    test_control = test.cp_type == "ctl_vehicle"
    test.drop("cp_type", axis=1, inplace=True)
    # 2. Standard scale continuous features [train + test non-control]
    scaler = StandardScaler()
    scaler.fit(pd.concat([X.iloc[:, 2:], test.loc[~test_control].iloc[:, 2:]]))
    X.iloc[:, 2:] = scaler.transform(X.iloc[:, 2:])
    test.iloc[:, 2:] = scaler.transform(test.iloc[:, 2:])
    # 3. Encode cp_time & cp_dose
    X["cp_dose"] = X["cp_dose"].map({"D1": 0, "D2": 1})
    X["cp_time"] = X["cp_time"] // 24 - 1
    test["cp_dose"] = test["cp_dose"].map({"D1": 0, "D2": 1})
    test["cp_time"] = test["cp_time"] // 24 - 1
    # 4. Shuffle data
    X = X.sample(frac=1.0, random_state=random_state)
    y = y.loc[X.index, :]
    assert np.all(y.index == X.index)
    return X, y, test, test_control


def split(X, y, n_folds=6, holdout=True):
    # remove too rare classes
    rare_columns = y.loc[:, y.sum(axis=0) < n_folds].columns
    y.drop(rare_columns, axis=1, inplace=True)
    # build stratified split index
    split_index = np.zeros(len(X)) - 1
    cv = MultilabelStratifiedKFold(n_splits=n_folds)
    for fold, (index_train, index_test) in enumerate(cv.split(X, y)):
        split_index[index_test] = fold
    classnames = y.columns.tolist()
    X, y = X.values, y.values
    # make holdout
    X_holdout, y_holdout = None, None
    if holdout:
        X_holdout, y_holdout = (
            X[split_index == n_folds - 1].copy(),
            y[split_index == n_folds - 1].copy(),
        )
        X, y = X[split_index < n_folds - 1], y[split_index < n_folds - 1]
        split_index = split_index[split_index < n_folds - 1]
    return X, y, split_index, X_holdout, y_holdout, classnames


def logloss(y_true, y_pred, eps=1e-15):
    y_pred[y_pred == 0] = eps
    y_pred[y_pred == 1] = 1 - eps
    return -(y_true * np.log(y_pred) + (1 - y_true) * (np.log(1 - y_pred))).mean()


class OOFTrainer:
    def __init__(
        self, estimator, metric, hparams=dict(), fit_params=dict(), predict_proba=True
    ):
        """
        Out-of-fold trainer
        Training data doesn't persisted
        Args:
        metric  -- function computing loss/metric
        hparams -- estimator hyperparameters
        """
        self.estimator = estimator
        self.metric = metric
        self.hparams = hparams
        self.fit_params = fit_params
        self.predict_proba = predict_proba
        self.models_ = None
        self.prediction_ = None
        self.valid_loss_ = None

    def finetune_hparams(self, grid, X, y):
        """Hyperparameters grid search"""
        pass

    def fit(self, X, y, split_index, verbose=True):
        """Fit models on each fold, validate and predict folds"""
        assert len(X) == len(y) == len(split_index)
        assert len(y.shape) > 1
        self.models_, self.valid_loss_ = [], []
        cv_prediction = np.zeros_like(y, dtype=np.float64)
        for fold in np.unique(split_index):
            if verbose:
                print(f"Fold {int(fold)}")
            # train|valid split
            train, valid = split_index != fold, split_index == fold
            x_train, y_train, x_valid, y_valid = X[train], y[train], X[valid], y[valid]
            # build and fit model
            model = clone(self.estimator)
            model = model.set_params(**self.hparams)
            model.fit(x_train, y_train, **self.fit_params)
            # predict and evaluate validation fold
            if self.predict_proba:
                prediction = model.predict_proba(x_valid)
            else:
                prediction = model.predict(x_valid)
            cv_prediction[valid] = prediction
            # save result
            self.models_.append(model)
            loss = self.metric(y_valid, prediction)
            self.valid_loss_.append(loss)
            if verbose:
                print(f"Loss={loss}")
        self.prediction_ = cv_prediction
        return self

    def predict(self, X):
        """Predict new data by all models and average"""
        prediction = np.zeros((len(X), self.prediction_.shape[1]), dtype=np.float64)
        for model in self.models_:
            prediction += model.predict_proba(X)
        prediction /= len(self.models_)
        return prediction


X, y, y_nonscored, test, submission = load_datasets("../input/lish-moa")
X, y, test, test_control = preprocess(X, y, test)
X, y, split_index, X_holdout, y_holdout, classnames = split(
    X, y, n_folds=5, holdout=False
)
# X, y, split_index, X_holdout, y_holdout, classnames = split(X, y, n_folds=6, holdout=True)
print("train  :", X.shape, y.shape, split_index.shape)
if X_holdout:
    print("holdout:", X_holdout.shape, y_holdout.shape)
print("test   :", test.shape, submission.shape)


# multioutput wrapper
class MultiLabelClassfifier(MultiOutputClassifier):
    def _predict_proba(self, X):
        results = [estimator.predict_proba(X) for estimator in self.estimators_]
        return np.moveaxis(results, 0, 1)[:, :, 1]


pca = PCA(n_components=50)
logistic = LogisticRegression(max_iter=150, tol=0.1, C=0.01, verbose=0)
base_model = Pipeline(steps=[("pca", pca), ("logistic", logistic)])
multilabel_model = MultiLabelClassfifier(base_model, n_jobs=-1)
oof_model = OOFTrainer(multilabel_model, logloss)
oof_model.fit(X, y, split_index, verbose=True)
result.append((p, np.mean(oof_model.valid_loss_)))
# todo: different hparams for different classes to MultiLabel
# from sklearn.base import BaseEstimator
# class MultiLabel(BaseEstimator):
#     def __init__(self, estimator, hparams, verbose=0):
#         """ """
#         self.estimator = estimator
#         self.hparams = hparams
#         self.verbose = verbose
#         self.models_ = []
#     def fit(self, X, y, **fit_params):
#         labels_iterator = range(y.shape[1])
#         for target in (tqdm(labels_iterator) if self.verbose else labels_iterator):
#             model = self.estimator(**self.hparams)
#             model.fit(X, y[:, target], **fit_params)
#             self.models_.append(model)
#         return self
#     def predict(self, X):
#         raise NotImplementedError
#     def predict_proba(self, X):
#         results = [model.predict_proba(X) for model in self.models_]
#         return np.moveaxis(results, 0, 1)[:, :, 1]
# pca = PCA(n_components=50)
# X_pca = pca.fit_transform(X)
# multilabel_lr = MultiLabel(LogisticRegression, dict(max_iter=300, tol=0.01, C=0.2, verbose=0), 1)
# multilabel_lr.fit(X_pca[split_index>0], y[split_index>0])
# logloss(y[split_index==0], multilabel_lr.predict_proba(X_pca[split_index==0]))
# def grid_search(estimator, hparams):
#     multilabel_lr = MultiLabel(estimator, hparams, verbose=1)
#     multilabel_lr.fit(X_pca[split_index>0], y[split_index>0])
#     loss = logloss(y[split_index==0], multilabel_lr.predict_proba(X_pca[split_index==0]))
#     return loss
# result = []
# grid = [1e-5, 1e-4, 1e-3, 1e-2]
# for p in grid:
#     loss = grid_search(LogisticRegression,
#                        dict(max_iter=100, tol=0.1, C=p, verbose=0))
#     result.append((p, loss))
# save result
prediction = oof_model.predict(test)
submission[:] = 0.0
for i, c in enumerate(classnames):
    submission[c] = prediction[:, i]
submission[test_control] = 0.0
submission.to_csv("submission.csv")
<|endoftext|>
[{"Id": 42448247, "ScriptId": 11620414, "ParentScriptVersionId": NaN, "ScriptLanguageId": 9, "AuthorUserId": 1309204, "CreationDate": "09/11/2020 11:59:42", "VersionNumber": 7.0, "Title": "MoA :: OOF & non-DL models", "EvaluationDate": "09/11/2020", "IsChange": true, "TotalLines": 241.0, "LinesInsertedFromPrevious": 54.0, "LinesChangedFromPrevious": 0.0, "LinesUnchangedFromPrevious": 187.0, "LinesInsertedFromFork": NaN, "LinesDeletedFromFork": NaN, "LinesChangedFromFork": NaN, "LinesUnchangedFromFork": NaN, "TotalVotes": 0}]
import os
import warnings
from tqdm import tqdm
import pandas as pd
import numpy as np
from ml_stratifiers import MultilabelStratifiedKFold
from sklearn.preprocessing import StandardScaler
from sklearn.base import clone
from sklearn.linear_model import LogisticRegression
from sklearn.multioutput import MultiOutputClassifier
from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA

warnings.filterwarnings("ignore")


def load_datasets(data_path):
    train_features = pd.read_csv(os.path.join(data_path, "train_features.csv"))
    train_targets_s = pd.read_csv(os.path.join(data_path, "train_targets_scored.csv"))
    train_targets_n = pd.read_csv(
        os.path.join(data_path, "train_targets_nonscored.csv")
    )
    test_features = pd.read_csv(os.path.join(data_path, "test_features.csv"))
    sample_submission = pd.read_csv(os.path.join(data_path, "sample_submission.csv"))
    train_features.set_index("sig_id", inplace=True)
    train_targets_s.set_index("sig_id", inplace=True)
    test_features.set_index("sig_id", inplace=True)
    sample_submission.set_index("sig_id", inplace=True)
    assert np.all(test_features.index == sample_submission.index)
    return (
        train_features,
        train_targets_s,
        train_targets_n,
        test_features,
        sample_submission,
    )


def preprocess(X, y, test, random_state=2020):
    X, y, test = X.copy(), y.copy(), test.copy()
    # 1. Filter cp_type=ctl_vehicle
    X = X[X.cp_type != "ctl_vehicle"]
    X.drop("cp_type", axis=1, inplace=True)
    test_control = test.cp_type == "ctl_vehicle"
    test.drop("cp_type", axis=1, inplace=True)
    # 2. Standard scale continuous features [train + test non-control]
    scaler = StandardScaler()
    scaler.fit(pd.concat([X.iloc[:, 2:], test.loc[~test_control].iloc[:, 2:]]))
    X.iloc[:, 2:] = scaler.transform(X.iloc[:, 2:])
    test.iloc[:, 2:] = scaler.transform(test.iloc[:, 2:])
    # 3. Encode cp_time & cp_dose
    X["cp_dose"] = X["cp_dose"].map({"D1": 0, "D2": 1})
    X["cp_time"] = X["cp_time"] // 24 - 1
    test["cp_dose"] = test["cp_dose"].map({"D1": 0, "D2": 1})
    test["cp_time"] = test["cp_time"] // 24 - 1
    # 4. Shuffle data
    X = X.sample(frac=1.0, random_state=random_state)
    y = y.loc[X.index, :]
    assert np.all(y.index == X.index)
    return X, y, test, test_control


def split(X, y, n_folds=6, holdout=True):
    # remove too rare classes
    rare_columns = y.loc[:, y.sum(axis=0) < n_folds].columns
    y.drop(rare_columns, axis=1, inplace=True)
    # build stratified split index
    split_index = np.zeros(len(X)) - 1
    cv = MultilabelStratifiedKFold(n_splits=n_folds)
    for fold, (index_train, index_test) in enumerate(cv.split(X, y)):
        split_index[index_test] = fold
    classnames = y.columns.tolist()
    X, y = X.values, y.values
    # make holdout
    X_holdout, y_holdout = None, None
    if holdout:
        X_holdout, y_holdout = (
            X[split_index == n_folds - 1].copy(),
            y[split_index == n_folds - 1].copy(),
        )
        X, y = X[split_index < n_folds - 1], y[split_index < n_folds - 1]
        split_index = split_index[split_index < n_folds - 1]
    return X, y, split_index, X_holdout, y_holdout, classnames


def logloss(y_true, y_pred, eps=1e-15):
    y_pred[y_pred == 0] = eps
    y_pred[y_pred == 1] = 1 - eps
    return -(y_true * np.log(y_pred) + (1 - y_true) * (np.log(1 - y_pred))).mean()


class OOFTrainer:
    def __init__(
        self, estimator, metric, hparams=dict(), fit_params=dict(), predict_proba=True
    ):
        """
        Out-of-fold trainer
        Training data doesn't persisted
        Args:
        metric  -- function computing loss/metric
        hparams -- estimator hyperparameters
        """
        self.estimator = estimator
        self.metric = metric
        self.hparams = hparams
        self.fit_params = fit_params
        self.predict_proba = predict_proba
        self.models_ = None
        self.prediction_ = None
        self.valid_loss_ = None

    def finetune_hparams(self, grid, X, y):
        """Hyperparameters grid search"""
        pass

    def fit(self, X, y, split_index, verbose=True):
        """Fit models on each fold, validate and predict folds"""
        assert len(X) == len(y) == len(split_index)
        assert len(y.shape) > 1
        self.models_, self.valid_loss_ = [], []
        cv_prediction = np.zeros_like(y, dtype=np.float64)
        for fold in np.unique(split_index):
            if verbose:
                print(f"Fold {int(fold)}")
            # train|valid split
            train, valid = split_index != fold, split_index == fold
            x_train, y_train, x_valid, y_valid = X[train], y[train], X[valid], y[valid]
            # build and fit model
            model = clone(self.estimator)
            model = model.set_params(**self.hparams)
            model.fit(x_train, y_train, **self.fit_params)
            # predict and evaluate validation fold
            if self.predict_proba:
                prediction = model.predict_proba(x_valid)
            else:
                prediction = model.predict(x_valid)
            cv_prediction[valid] = prediction
            # save result
            self.models_.append(model)
            loss = self.metric(y_valid, prediction)
            self.valid_loss_.append(loss)
            if verbose:
                print(f"Loss={loss}")
        self.prediction_ = cv_prediction
        return self

    def predict(self, X):
        """Predict new data by all models and average"""
        prediction = np.zeros((len(X), self.prediction_.shape[1]), dtype=np.float64)
        for model in self.models_:
            prediction += model.predict_proba(X)
        prediction /= len(self.models_)
        return prediction


X, y, y_nonscored, test, submission = load_datasets("../input/lish-moa")
X, y, test, test_control = preprocess(X, y, test)
X, y, split_index, X_holdout, y_holdout, classnames = split(
    X, y, n_folds=5, holdout=False
)
# X, y, split_index, X_holdout, y_holdout, classnames = split(X, y, n_folds=6, holdout=True)
print("train  :", X.shape, y.shape, split_index.shape)
if X_holdout:
    print("holdout:", X_holdout.shape, y_holdout.shape)
print("test   :", test.shape, submission.shape)


# multioutput wrapper
class MultiLabelClassfifier(MultiOutputClassifier):
    def _predict_proba(self, X):
        results = [estimator.predict_proba(X) for estimator in self.estimators_]
        return np.moveaxis(results, 0, 1)[:, :, 1]


pca = PCA(n_components=50)
logistic = LogisticRegression(max_iter=150, tol=0.1, C=0.01, verbose=0)
base_model = Pipeline(steps=[("pca", pca), ("logistic", logistic)])
multilabel_model = MultiLabelClassfifier(base_model, n_jobs=-1)
oof_model = OOFTrainer(multilabel_model, logloss)
oof_model.fit(X, y, split_index, verbose=True)
result.append((p, np.mean(oof_model.valid_loss_)))
# todo: different hparams for different classes to MultiLabel
# from sklearn.base import BaseEstimator
# class MultiLabel(BaseEstimator):
#     def __init__(self, estimator, hparams, verbose=0):
#         """ """
#         self.estimator = estimator
#         self.hparams = hparams
#         self.verbose = verbose
#         self.models_ = []
#     def fit(self, X, y, **fit_params):
#         labels_iterator = range(y.shape[1])
#         for target in (tqdm(labels_iterator) if self.verbose else labels_iterator):
#             model = self.estimator(**self.hparams)
#             model.fit(X, y[:, target], **fit_params)
#             self.models_.append(model)
#         return self
#     def predict(self, X):
#         raise NotImplementedError
#     def predict_proba(self, X):
#         results = [model.predict_proba(X) for model in self.models_]
#         return np.moveaxis(results, 0, 1)[:, :, 1]
# pca = PCA(n_components=50)
# X_pca = pca.fit_transform(X)
# multilabel_lr = MultiLabel(LogisticRegression, dict(max_iter=300, tol=0.01, C=0.2, verbose=0), 1)
# multilabel_lr.fit(X_pca[split_index>0], y[split_index>0])
# logloss(y[split_index==0], multilabel_lr.predict_proba(X_pca[split_index==0]))
# def grid_search(estimator, hparams):
#     multilabel_lr = MultiLabel(estimator, hparams, verbose=1)
#     multilabel_lr.fit(X_pca[split_index>0], y[split_index>0])
#     loss = logloss(y[split_index==0], multilabel_lr.predict_proba(X_pca[split_index==0]))
#     return loss
# result = []
# grid = [1e-5, 1e-4, 1e-3, 1e-2]
# for p in grid:
#     loss = grid_search(LogisticRegression,
#                        dict(max_iter=100, tol=0.1, C=p, verbose=0))
#     result.append((p, loss))
# save result
prediction = oof_model.predict(test)
submission[:] = 0.0
for i, c in enumerate(classnames):
    submission[c] = prediction[:, i]
submission[test_control] = 0.0
submission.to_csv("submission.csv")
