_1_1_pre_process_dfal.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T01:53:26.261181Z",
     "start_time": "2018-04-20T01:53:25.309381Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T01:53:26.309411Z",
     "start_time": "2018-04-20T01:53:26.263669Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_dfal():\n",
    "    dump_nominal_file = os.path.join(utils.cache_root, 'dfda_nominal.pkl')\n",
    "    dump_textual_file = os.path.join(utils.cache_root, 'dfda_textual.pkl')\n",
    "    if not os.path.exists(dump_nominal_file):\n",
    "        tr = pd.read_csv('./input/round1_ijcai_18_train_20180301.txt', sep=' ', dtype={'is_trade':np.uint8})\n",
    "        tr.is_trade = tr.is_trade.astype(np.int8)\n",
    "        te = pd.read_csv('./input/round1_ijcai_18_test_b_20180418.txt', sep=' ')\n",
    "        da = pd.concat([tr, te], axis=0)\n",
    "        da = utils.add_time_fields(da)\n",
    "        \n",
    "        for col in utils.nominal_cate_cols + utils.identity_cols:\n",
    "            da[col] = LabelEncoder().fit_transform(da[col])\n",
    "        \n",
    "        for col in utils.ordinal_cate_cols:\n",
    "            levels = sorted(da[col].unique())\n",
    "            da[col] = da[col].apply(lambda x : levels.index(x)).astype(np.uint8)\n",
    "        \n",
    "        del da['context_id']\n",
    "        del da['context_timestamp']\n",
    "        del da['ts']\n",
    "        da, _ = utils.reduce_mem_usage(da)\n",
    "        utils.dump_pickle(da[utils.textual_cols], dump_textual_file)\n",
    "        utils.dump_pickle(da.drop(utils.textual_cols, axis=1), dump_nominal_file)\n",
    "    print('gen dfal ok.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T01:54:00.153609Z",
     "start_time": "2018-04-20T01:53:26.793806Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is : 91.92448616027832  MB\n",
      "___MEMORY USAGE AFTER COMPLETION:___\n",
      "Memory usage is:  41.738685607910156  MB\n",
      "This is  45.4054054054054 % of the initial size\n",
      "gen dfal ok.\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    gen_dfal()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "stats36",
   "language": "python",
   "name": "stats36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}