[{'role': 'system', 'content': "You are an expert in machine learning and your task is to classify code snippets into one of the following categories: ['helper_functions', 'load_data', 'data_exploration', 'data_preprocessing', 'evaluation', 'modelling', 'prediction', 'result_visualization', 'save_results', 'comment_only']."}, {'role': 'user', 'content': 'Identify the single most appropriate category to which the provided code belongs. Provide your response as a single category label without any additional information, commentary, or personal opinions.\\nPython Code:import gc\nimport numpy as np\nimport pandas as pd\nimport xgboost as xgb\nfrom sklearn.cross_validation import StratifiedKFold\nfrom sklearn.metrics import matthews_corrcoef\nfrom operator import itemgetter\n\n# per raddar, all date features except for stations 24+25 are identical\n\n\n\n\n\ndef get_date_features():\n    directory = \'../input/\'\n    trainfile = \'train_date.csv\'\n    \n\n    for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n                                          chunksize=1,\n                                          low_memory=False)):\n        features = list(chunk.columns)\n        break\n\n    seen = np.zeros(52)\n    rv = []\n    for f in features:\n        if f == \'Id\':\n            rv.append(f)\n            continue\n            \n        station = int(f.split(\'_\')[1][1:])\n        \n        if seen[station]:\n            continue\n        \n        seen[station] = 1\n        rv.append(f)\n        \n    return rv\n        \nusefuldatefeatures = get_date_features()\n\ndef get_mindate():\n    directory = \'../input/\'\n    trainfile = \'train_date.csv\'\n    testfile = \'test_date.csv\'\n    \n    features = None\n    subset = None\n    \n    for i, chunk in enumerate(pd.read_csv(directory + trainfile,\n                                          usecols=usefuldatefeatures,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        print(i)\n        \n        if features is None:\n            features = list(chunk.columns)\n            features.remove(\'Id\')\n        \n        df_mindate_chunk = chunk[[\'Id\']].copy()\n        df_mindate_chunk[\'mindate\'] = chunk[features].min(axis=1).values\n        \n        if subset is None:\n            subset = df_mindate_chunk.copy()\n        else:\n            subset = pd.concat([subset, df_mindate_chunk])\n            \n        del chunk\n\n        gc.collect()\n\n    for i, chunk in enumerate(pd.read_csv(directory + testfile,\n                                          usecols=usefuldatefeatures,\n                                          chunksize=50000,\n                                          low_memory=False)):\n        print(i)\n        \n        df_mindate_chunk = chunk[[\'Id\']].copy()\n        df_mindate_chunk[\'mindate\'] = chunk[features].min(axis=1).values\n        subset = pd.concat([subset, df_mindate_chunk])\n        \n        del chunk\n        gc.collect()      \n        \n    return subset\n\n\ndf_mindate = get_mindate()\n\ndf_mindate.sort_values(by=[\'mindate\', \'Id\'], inplace=True)\n\ndf_mindate[\'mindate_id_diff\'] = df_mindate.Id.diff()\n\nmidr = np.full_like(df_mindate.mindate_id_diff.values, np.nan)\nmidr[0:-1] = -df_mindate.mindate_id_diff.values[1:]\n\ndf_mindate[\'mindate_id_diff_reverse\'] = midr\n\n\n\n\ndef mcc(tp, tn, fp, fn):\n    sup = tp * tn - fp * fn\n    inf = (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)\n    if inf == 0:\n        return 0\n    else:\n        return sup / np.sqrt(inf)\n\n\ndef eval_mcc(y_true, y_prob, show=False):\n    idx = np.argsort(y_prob)\n    y_true_sort = y_true[idx]\n    n = y_true.shape[0]\n    nump = 1.0 * np.sum(y_true)  # number of positive\n    numn = n - nump  # number of negative\n    tp = nump\n    tn = 0.0\n    fp = numn\n    fn = 0.0\n    best_mcc = 0.0\n    best_id = -1\n    mccs = np.zeros(n)\n    for i in range(n):\n        if y_true_sort[i] == 1:\n            tp -= 1.0\n            fn += 1.0\n        else:\n            fp -= 1.0\n            tn += 1.0\n        new_mcc = mcc(tp, tn, fp, fn)\n        mccs[i] = new_mcc\n        if new_mcc >= best_mcc:\n            best_mcc = new_mcc\n            best_id = i\n    if show:\n        best_proba = y_prob[idx[best_id]]\n        y_pred = (y_prob > best_proba).astype(int)\n        return best_proba, best_mcc, y_pred\n    else:\n        return best_mcc\n\n\ndef mcc_eval(y_prob, dtrain):\n    y_true = dtrain.get_label()\n    best_mcc = eval_mcc(y_true, y_prob)\n    return \'MCC\', best_mcc\n\n\ndef create_feature_map(features):\n    outfile = open(\'xgb.fmap\', \'w\')\n    for i, feat in enumerate(features):\n        outfile.write(\'{0}\\\\t{1}\\\\tq\\\n\'.format(i, feat))\n    outfile.close()\n\n\ndef get_importance(gbm, features):\n    create_feature_map(features)\n    importance = gbm.get_fscore(fmap=\'xgb.fmap\')\n    importance = sorted(importance.items(), key=itemgetter(1), reverse=True)\n    return importance\n\n\ndef LeaveOneOut(data1, data2, columnName, useLOO=False):\n    grpOutcomes = data1.groupby(columnName)[\'Response\'].mean().reset_index()\n    grpCount = data1.groupby(columnName)[\'Response\'].count().reset_index()\n    grpOutcomes[\'cnt\'] = grpCount.Response\n    if(useLOO):\n        grpOutcomes = grpOutcomes[grpOutcomes.cnt > 1]\n    grpOutcomes.drop(\'cnt\', inplace=True, axis=1)\n    outcomes = data2[\'Response\'].values\n    x = pd.merge(data2[[columnName, \'Response\']], grpOutcomes,\n                 suffixes=(\'x_\', \'\'),\n                 how=\'left\',\n                 on=columnName,\n                 left_index=True)[\'Response\']\n    if(useLOO):\n        x = ((x*x.shape[0])-outcomes)/(x.shape[0]-1)\n        #  x = x + np.random.normal(0, .01, x.shape[0])\n    return x.fillna(x.mean())\n\n\ndef GrabData():\n    directory = \'../input/\'\n    trainfiles = [\'train_categorical.csv\',\n                  \'train_date.csv\',\n                  \'train_numeric.csv\']\n    testfiles = [\'test_categorical.csv\',\n                 \'test_date.csv\',\n                 \'test_numeric.csv\']\n\n    cols = [[\'Id\',\n             \'L1_S24_F1559\', \'L3_S32_F3851\',\n             \'L1_S24_F1827\', \'L1_S24_F1582\',\n             \'L3_S32_F3854\', \'L1_S24_F1510\',\n             \'L1_S24_F1525\', \'L2_S26_F3099\'],\n            [\'Id\',\n             \'L3_S30_D3496\', \'L3_S30_D3506\',\n             \'L3_S30_D3501\', \'L3_S30_D3516\',\n             \'L3_S30_D3511\', \'L3_S32_D3852\',\n             \'L3_S33_D3858\', \'L3_S34_D3875\',\n             \'L3_S29_D3316\'],\n            [\'Id\',\n             \'L1_S24_F1846\', \'L3_S32_F3850\',\n             \'L1_S24_F1695\', \'L1_S24_F1632\',\n             \'L3_S33_F3855\', \'L1_S24_F1604\',\n             \'L3_S29_F3407\', \'L3_S33_F3865\',\n             \'L3_S38_F3952\', \'L1_S24_F1723\',\n             \'L3_S33_F3861\', \'L3_S33_F3857\',\n             \'L3_S33_F3859\', \'L3_S34_F3876\', \'L3_S29_F3461\',\n             \'Response\']]\n    traindata = None\n    testdata = None\n\n    for i, f in enumerate(trainfiles):\n        print(f)\n        subset = None\n        for i, chunk in enumerate(pd.read_csv(directory + f,\n                                              usecols=cols[i],\n                                              chunksize=50000,\n                                              low_memory=False)):\n\n            print(i)\n            if subset is None:\n                subset = chunk.copy()\n            else:\n                subset = pd.concat([subset, chunk])\n            del chunk\n            gc.collect()\n        if traindata is None:\n            traindata = subset.copy()\n        else:\n            traindata = pd.merge(traindata, subset.copy(), on="Id")\n        del subset\n        gc.collect()\n    del cols[2][-1]  # Test doesn\'t have response!\n    for i, f in enumerate(testfiles):\n        print(f)\n        subset = None\n        for i, chunk in enumerate(pd.read_csv(directory + f,\n                                              usecols=cols[i],\n                                              chunksize=50000,\n                                              low_memory=False)):\n            print(i)\n            if subset is None:\n                subset = chunk.copy()\n            else:\n                subset = pd.concat([subset, chunk])\n            del chunk\n            gc.collect()\n        if testdata is None:\n            testdata = subset.copy()\n        else:\n            testdata = pd.merge(testdata, subset.copy(), on="Id")\n        del subset\n        gc.collect()\n        \n    traindata = traindata.merge(df_mindate, on=\'Id\')\n    testdata = testdata.merge(df_mindate, on=\'Id\')\n        \n    testdata[\'Response\'] = 0  # Add Dummy Value\n    visibletraindata = traindata[::2]\n    blindtraindata = traindata[1::2]\n    print(blindtraindata.columns)\n    for i in range(2):\n        for col in cols[i][1:]:\n            print(col)\n            blindtraindata.loc[:, col] = LeaveOneOut(visibletraindata,\n                                                     blindtraindata,\n                                                     col, False).values\n            testdata.loc[:, col] = LeaveOneOut(visibletraindata,\n                                               testdata, col, False).values\n    del visibletraindata\n    gc.collect()\n    testdata.drop(\'Response\', inplace=True, axis=1)\n    return blindtraindata, testdata\n\n\ndef Train():\n    train, test = GrabData()\n    print(\'Train:\', train.shape)\n    print(\'Test\', test.shape)\n    features = list(train.columns)\n    features.remove(\'Response\')\n    features.remove(\'Id\')\n    print(features)\n    num_rounds = 50\n    params = {}\n    params[\'objective\'] = "binary:logistic"\n    params[\'eta\'] = 0.021\n    params[\'max_depth\'] = 7\n    params[\'colsample_bytree\'] = 0.82\n    params[\'min_child_weight\'] = 3\n    params[\'base_score\'] = 0.005\n    params[\'silent\'] = True\n\n    print(\'Fitting\')\n    trainpredictions = None\n    testpredictions = None\n\n    dvisibletrain = \\\\\n        xgb.DMatrix(train[features],\n                    train.Response,\n                    silent=True)\n    dtest = \\\\\n        xgb.DMatrix(test[features],\n                    silent=True)\n\n    folds = 1\n    for i in range(folds):\n        print(\'Fold:\', i)\n        params[\'seed\'] = i\n        watchlist = [(dvisibletrain, \'train\'), (dvisibletrain, \'val\')]\n        clf = xgb.train(params, dvisibletrain,\n                        num_boost_round=num_rounds,\n                        evals=watchlist,\n                        early_stopping_rounds=20,\n                        feval=mcc_eval,\n                        maximize=True\n                        )\n        limit = clf.best_iteration+1\n        # limit = clf.best_ntree_limit\n        predictions = \\\\\n            clf.predict(dvisibletrain, ntree_limit=limit)\n\n        best_proba, best_mcc, y_pred = eval_mcc(train.Response,\n                                                predictions,\n                                                True)\n        print(\'tree limit:\', limit)\n        print(\'mcc:\', best_mcc)\n        print(matthews_corrcoef(train.Response,\n                                y_pred))\n        if(trainpredictions is None):\n            trainpredictions = predictions\n        else:\n            trainpredictions += predictions\n        predictions = clf.predict(dtest, ntree_limit=limit)\n        if(testpredictions is None):\n            testpredictions = predictions\n        else:\n            testpredictions += predictions\n        imp = get_importance(clf, features)\n        print(\'Importance array: \', imp)\n\n    best_proba, best_mcc, y_pred = eval_mcc(train.Response,\n                                            trainpredictions/folds,\n                                            True)\n    print(matthews_corrcoef(train.Response,\n                            y_pred))\n\n    submission = pd.DataFrame({"Id": train.Id,\n                               "Prediction": trainpredictions/folds,\n                               "Response": train.Response})\n    submission[[\'Id\',\n                \'Prediction\',\n                \'Response\']].to_csv(\'rawtrainxgbsubmission\'+str(folds)+\'.csv\',\n                                    index=False)\n\n    submission = pd.DataFrame({"Id": test.Id.values,\n                               "Response": testpredictions/folds})\n    submission[[\'Id\', \'Response\']].to_csv(\'rawxgbsubmission\'+str(folds)+\'.csv\',\n                                          index=False)\n    y_pred = (testpredictions/folds > .22).astype(int)\n    submission = pd.DataFrame({"Id": test.Id.values,\n                               "Response": y_pred})\n    submission[[\'Id\', \'Response\']].to_csv(\'xgbsubmission\'+str(folds)+\'.csv\',\n                                          index=False)\n\nif __name__ == "__main__":\n    print(\'Started\')\n    Train()\n    print(\'Finished\')\n    \''}, {'role': 'assistant', 'content': 'helper_functions'}]