<kaggle_start><code>import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
import os

for dirname, _, filenames in os.walk("/kaggle/input"):
    for filename in filenames:
        print(os.path.join(dirname, filename))
# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All"
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
X = pd.read_csv("/kaggle/input/playground-series-s3e18/train.csv")
# Drop EC3-6 to prevent leakage
X = X.drop(["id", "EC3", "EC4", "EC5", "EC6"], axis=1)
# remove -666 row
idxs = X[
    (X["FpDensityMorgan1"] == -666)
    | (X["FpDensityMorgan2"] == -666)
    | (X["FpDensityMorgan3"] == -666)
].index
X = X.drop(idxs, axis=0)
y1 = X.pop("EC1")
y2 = X.pop("EC2")
X.shape

# I decide to trian models on EC1 and EC2 separately. The objective is to maximize the cv score.
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import BaggingClassifier, VotingClassifier
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, FunctionTransformer

# Use optuna to tune the hp
import optuna


def add_features():
    # add several features
    pass


# # EC1
# Use LGBM to predict EC1
from lightgbm import LGBMClassifier

param1_distributions = {
    "learning_rate": optuna.distributions.FloatDistribution(1e-3, 0.05),
    "n_estimators": optuna.distributions.IntDistribution(80, 500),
    "min_child_samples": optuna.distributions.IntDistribution(10, 90),
    "reg_alpha": optuna.distributions.FloatDistribution(1e-3, 10.0),
    "subsample": optuna.distributions.FloatDistribution(0.6, 1.0),
    "colsample_bytree": optuna.distributions.FloatDistribution(0.6, 1.0),
}
optuna_search1 = optuna.integration.OptunaSearchCV(
    LGBMClassifier(random_state=100),
    param1_distributions,
    scoring="roc_auc",
    refit=True,
    random_state=1289,
    verbose=0,
)
# optuna_search1.fit(X,y1)
# Tuned by optuna
clf1 = LGBMClassifier(
    random_state=100,
    n_estimators=160,
    learning_rate=0.04,
    min_child_samples=50,
    reg_alpha=1.06,
    reg_lambda=2,
)
gcv1 = GridSearchCV(clf1, {}, refit=True, scoring="roc_auc")
gcv1.fit(X, y1)
gcv1.best_score_

# # EC2
from sklearn.neighbors import KNeighborsClassifier

model = make_pipeline(
    StandardScaler(),
    BaggingClassifier(
        KNeighborsClassifier(n_neighbors=500, weights="distance"),
        n_estimators=100,
        bootstrap=False,
        max_features=9,
        random_state=1,
    ),
)
gcv2 = GridSearchCV(model, {}, refit=True, scoring="roc_auc")
gcv2.fit(X, y2)
gcv2.best_score_

# # Submission
X_test = pd.read_csv("/kaggle/input/playground-series-s3e18/test.csv")
ids = X_test.pop("id")
y1_test = gcv1.predict_proba(X_test)[:, 1]
y2_test = gcv2.predict_proba(X_test)[:, 1]
submit = pd.DataFrame({"id": ids, "EC1": y1_test, "EC2": y2_test})
submit.to_csv("submission.csv", index=False)
<|endoftext|>
[{"Id": 135796633, "ScriptId": 40539752, "ParentScriptVersionId": NaN, "ScriptLanguageId": 9, "AuthorUserId": 13503132, "CreationDate": "07/05/2023 10:09:03", "VersionNumber": 4.0, "Title": "Enzyme\ud83e\udda0", "EvaluationDate": "07/05/2023", "IsChange": true, "TotalLines": 115.0, "LinesInsertedFromPrevious": 74.0, "LinesChangedFromPrevious": 0.0, "LinesUnchangedFromPrevious": 41.0, "LinesInsertedFromFork": NaN, "LinesDeletedFromFork": NaN, "LinesChangedFromFork": NaN, "LinesUnchangedFromFork": NaN, "TotalVotes": 0}]
import numpy as np  # linear algebra
import pandas as pd  # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory
import os

for dirname, _, filenames in os.walk("/kaggle/input"):
    for filename in filenames:
        print(os.path.join(dirname, filename))
# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All"
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
X = pd.read_csv("/kaggle/input/playground-series-s3e18/train.csv")
# Drop EC3-6 to prevent leakage
X = X.drop(["id", "EC3", "EC4", "EC5", "EC6"], axis=1)
# remove -666 row
idxs = X[
    (X["FpDensityMorgan1"] == -666)
    | (X["FpDensityMorgan2"] == -666)
    | (X["FpDensityMorgan3"] == -666)
].index
X = X.drop(idxs, axis=0)
y1 = X.pop("EC1")
y2 = X.pop("EC2")
X.shape

# I decide to trian models on EC1 and EC2 separately. The objective is to maximize the cv score.
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import BaggingClassifier, VotingClassifier
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, FunctionTransformer

# Use optuna to tune the hp
import optuna


def add_features():
    # add several features
    pass


# # EC1
# Use LGBM to predict EC1
from lightgbm import LGBMClassifier

param1_distributions = {
    "learning_rate": optuna.distributions.FloatDistribution(1e-3, 0.05),
    "n_estimators": optuna.distributions.IntDistribution(80, 500),
    "min_child_samples": optuna.distributions.IntDistribution(10, 90),
    "reg_alpha": optuna.distributions.FloatDistribution(1e-3, 10.0),
    "subsample": optuna.distributions.FloatDistribution(0.6, 1.0),
    "colsample_bytree": optuna.distributions.FloatDistribution(0.6, 1.0),
}
optuna_search1 = optuna.integration.OptunaSearchCV(
    LGBMClassifier(random_state=100),
    param1_distributions,
    scoring="roc_auc",
    refit=True,
    random_state=1289,
    verbose=0,
)
# optuna_search1.fit(X,y1)
# Tuned by optuna
clf1 = LGBMClassifier(
    random_state=100,
    n_estimators=160,
    learning_rate=0.04,
    min_child_samples=50,
    reg_alpha=1.06,
    reg_lambda=2,
)
gcv1 = GridSearchCV(clf1, {}, refit=True, scoring="roc_auc")
gcv1.fit(X, y1)
gcv1.best_score_

# # EC2
from sklearn.neighbors import KNeighborsClassifier

model = make_pipeline(
    StandardScaler(),
    BaggingClassifier(
        KNeighborsClassifier(n_neighbors=500, weights="distance"),
        n_estimators=100,
        bootstrap=False,
        max_features=9,
        random_state=1,
    ),
)
gcv2 = GridSearchCV(model, {}, refit=True, scoring="roc_auc")
gcv2.fit(X, y2)
gcv2.best_score_

# # Submission
X_test = pd.read_csv("/kaggle/input/playground-series-s3e18/test.csv")
ids = X_test.pop("id")
y1_test = gcv1.predict_proba(X_test)[:, 1]
y2_test = gcv2.predict_proba(X_test)[:, 1]
submit = pd.DataFrame({"id": ids, "EC1": y1_test, "EC2": y2_test})
submit.to_csv("submission.csv", index=False)
