{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Instruction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To obtain the dataset Amazon used for algorithms comparison:\n",
    "\n",
    "1) Download file `train.csv` from Amazon Employee Acess Challenge competition on Kaggle: https://www.kaggle.com/c/amazon-employee-access-challenge/data\n",
    "\n",
    "2) Put it to the same directory as this notebook.\n",
    "\n",
    "3) Run all the cells of this notebook successively to produce files for training and testing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "resulting_train_filename = \"train\"\n",
    "resulting_test_filename = \"test\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ACTION</th>\n",
       "      <th>RESOURCE</th>\n",
       "      <th>MGR_ID</th>\n",
       "      <th>ROLE_ROLLUP_1</th>\n",
       "      <th>ROLE_ROLLUP_2</th>\n",
       "      <th>ROLE_DEPTNAME</th>\n",
       "      <th>ROLE_TITLE</th>\n",
       "      <th>ROLE_FAMILY_DESC</th>\n",
       "      <th>ROLE_FAMILY</th>\n",
       "      <th>ROLE_CODE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>39353</td>\n",
       "      <td>85475</td>\n",
       "      <td>117961</td>\n",
       "      <td>118300</td>\n",
       "      <td>123472</td>\n",
       "      <td>117905</td>\n",
       "      <td>117906</td>\n",
       "      <td>290919</td>\n",
       "      <td>117908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>17183</td>\n",
       "      <td>1540</td>\n",
       "      <td>117961</td>\n",
       "      <td>118343</td>\n",
       "      <td>123125</td>\n",
       "      <td>118536</td>\n",
       "      <td>118536</td>\n",
       "      <td>308574</td>\n",
       "      <td>118539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>36724</td>\n",
       "      <td>14457</td>\n",
       "      <td>118219</td>\n",
       "      <td>118220</td>\n",
       "      <td>117884</td>\n",
       "      <td>117879</td>\n",
       "      <td>267952</td>\n",
       "      <td>19721</td>\n",
       "      <td>117880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>36135</td>\n",
       "      <td>5396</td>\n",
       "      <td>117961</td>\n",
       "      <td>118343</td>\n",
       "      <td>119993</td>\n",
       "      <td>118321</td>\n",
       "      <td>240983</td>\n",
       "      <td>290919</td>\n",
       "      <td>118322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>42680</td>\n",
       "      <td>5905</td>\n",
       "      <td>117929</td>\n",
       "      <td>117930</td>\n",
       "      <td>119569</td>\n",
       "      <td>119323</td>\n",
       "      <td>123932</td>\n",
       "      <td>19793</td>\n",
       "      <td>119325</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ACTION  RESOURCE  MGR_ID  ROLE_ROLLUP_1  ROLE_ROLLUP_2  ROLE_DEPTNAME  \\\n",
       "0       1     39353   85475         117961         118300         123472   \n",
       "1       1     17183    1540         117961         118343         123125   \n",
       "2       1     36724   14457         118219         118220         117884   \n",
       "3       1     36135    5396         117961         118343         119993   \n",
       "4       1     42680    5905         117929         117930         119569   \n",
       "\n",
       "   ROLE_TITLE  ROLE_FAMILY_DESC  ROLE_FAMILY  ROLE_CODE  \n",
       "0      117905            117906       290919     117908  \n",
       "1      118536            118536       308574     118539  \n",
       "2      117879            267952        19721     117880  \n",
       "3      118321            240983       290919     118322  \n",
       "4      119323            123932        19793     119325  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "target = data[\"ACTION\"].apply(lambda x: 1 if x == 1 else -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.drop([\"ACTION\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32769, 9)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing train/test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_idx = pd.read_csv(\"stratified_train_idx.txt\", header=None)\n",
    "test_idx = pd.read_csv(\"stratified_test_idx.txt\", header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Xtrain = data.iloc[train_idx[0]]\n",
    "Ytrain = target.iloc[train_idx[0]]\n",
    "Xtest = data.iloc[test_idx[0]]\n",
    "Ytest = target.iloc[test_idx[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# creating file with features\n",
    "def prepare_pool(data, labels, filename):\n",
    "    X = data.values\n",
    "    y = labels.values\n",
    "    with open(filename, \"w\") as fout:\n",
    "        for i in range(data.shape[0]):\n",
    "            fout.write(str(y[i]) + \"\\t\" + \"\\t\".join(map(str, X[i])) + \"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "prepare_pool(Xtrain, Ytrain, resulting_train_filename)\n",
    "prepare_pool(Xtest, Ytest, resulting_test_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(resulting_train_filename + '.cd', 'w') as fout:\n",
    "    fout.write('0\\tTarget\\n')\n",
    "    for cat_f_id in xrange(data.shape[1]):\n",
    "        fout.write('{}\\tCateg\\n'.format(cat_f_id + 1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
