_2_7_gen_level_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T09:03:56.192535Z",
     "start_time": "2018-04-21T09:03:52.741007Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import gc\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from utils import (load_pickle, dump_pickle, get_nominal_dfal, feats_root,\n",
    "                   mem_usage, reduce_mem_usage, nominal_cate_cols,\n",
    "                   ordinal_cate_cols, identity_cols, continual_cols, top, freq,\n",
    "                   unique, vrange, percentile)\n",
    "\n",
    "pd.set_option('display.max_columns', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T09:03:56.218657Z",
     "start_time": "2018-04-21T09:03:56.196312Z"
    }
   },
   "outputs": [],
   "source": [
    "level_cols = [\n",
    "    'item_price_level', 'item_sales_level', 'item_collected_level',\n",
    "    'item_pv_level', 'user_age_level', 'user_star_level',\n",
    "    'shop_review_num_level', 'shop_star_level'\n",
    "]\n",
    "\n",
    "ignores = [\n",
    "    'mi_item_collected_level', 'user_occupation_id_item_collected_level',\n",
    "    'hm_item_price_level', 'item_category_list_shop_star_level',\n",
    "    'ho_user_star_level', 'ho_item_collected_level',\n",
    "    'item_city_id_user_age_level', 'hm_user_age_level', 'hm_item_sales_level',\n",
    "    'mi_user_star_level', 'user_gender_id_item_sales_level',\n",
    "    'item_city_id_shop_star_level', 'mi_user_age_level',\n",
    "    'item_city_id_user_star_level', 'item_brand_id_user_star_level',\n",
    "    'mi_item_sales_level', 'user_occupation_id_item_price_level',\n",
    "    'ho_item_pv_level', 'user_occupation_id_item_pv_level',\n",
    "    'mi_shop_review_num_level', 'mi_shop_star_level',\n",
    "    'item_category_list_user_age_level', 'ho_item_price_level',\n",
    "    'ho_item_sales_level', 'user_occupation_id_shop_review_num_level',\n",
    "    'user_gender_id_item_price_level', 'ho_user_age_level',\n",
    "    'item_brand_id_shop_review_num_level',\n",
    "    'user_occupation_id_item_sales_level',\n",
    "    'user_occupation_id_shop_star_level', 'user_gender_id_shop_star_level',\n",
    "    'hm_user_star_level', 'mi_item_price_level', 'mi_item_pv_level',\n",
    "    'item_city_id_shop_review_num_level', 'item_brand_id_user_age_level',\n",
    "    'item_brand_id_shop_star_level', 'ho_shop_review_num_level',\n",
    "    'user_gender_id_item_pv_level', 'user_gender_id_shop_review_num_level',\n",
    "    'ho_shop_star_level', 'item_category_list_user_star_level',\n",
    "    'user_gender_id_item_collected_level',\n",
    "    'item_category_list_shop_review_num_level'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T09:03:56.303798Z",
     "start_time": "2018-04-21T09:03:56.224163Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_level_agg_features(data, last_da, win_das, col):\n",
    "    agg_cols = list(filter(lambda x: not x.startswith(col[:4]), level_cols))\n",
    "    for c in ignores:\n",
    "        if c.startswith(col):\n",
    "            agg_cols.remove(c.replace(col+'_', ''))\n",
    "    \n",
    "    if len(agg_cols) == 0:\n",
    "        return None\n",
    "    \n",
    "    data = data.copy()\n",
    "    indexing = (data.da < last_da) & (data.da >= last_da - win_das)\n",
    "    gp = data.loc[indexing, [col] + agg_cols].groupby(col)[agg_cols]\n",
    "\n",
    "    aggs = gp.agg([\n",
    "        'mean', 'std', 'sem', pd.DataFrame.kurt, pd.DataFrame.skew,\n",
    "        pd.DataFrame.mad, freq,\n",
    "        percentile(.3),\n",
    "        percentile(.9)\n",
    "    ])\n",
    "    aggs.columns = [\n",
    "        'agg_level_{}_{}_{}_wd_{}'.format(col, c[0], c[1], win_das)\n",
    "        for c in aggs.columns\n",
    "    ]\n",
    "    aggs = aggs.reset_index()\n",
    "    data = data.loc[data.da == last_da].merge(aggs, how='left', on=col)\n",
    "    data.drop(level_cols, inplace=True, axis=1)\n",
    "    data.drop_duplicates([col, 'da'], inplace=True)\n",
    "    data.fillna(0, inplace=True)\n",
    "    data, _ = reduce_mem_usage(data)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T09:03:56.419760Z",
     "start_time": "2018-04-21T09:03:56.307683Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_level_aggs(col, updata=False):\n",
    "    feat_path = os.path.join(feats_root,'level_aggs_{}.pkl'.format(col))\n",
    "    if os.path.exists(feat_path) and updata == False:\n",
    "        print('Found ' + feat_path)\n",
    "    else:\n",
    "        print('Generating ' + feat_path)\n",
    "        dfal = get_nominal_dfal()[[col, 'da'] + level_cols]\n",
    "        dmax = dfal.da.max()\n",
    "        dmin = dfal.da.min()\n",
    "        \n",
    "        level_agg = None\n",
    "        for da in sorted(dfal.da.unique())[1:]:\n",
    "            da_agg = None\n",
    "            for win_das in [1, 2, 3]:\n",
    "                if da - win_das < dmin:\n",
    "                    continue\n",
    "                agg = gen_level_agg_features(dfal, da, win_das, col)\n",
    "                if agg is None:\n",
    "                    continue\n",
    "                print('Generated {} {} {}'.format(col, da, win_das))\n",
    "                if da_agg is None:\n",
    "                    da_agg = agg\n",
    "                else:\n",
    "                    da_agg = da_agg.merge(agg, how='outer')\n",
    "            if level_agg is None:\n",
    "                level_agg = da_agg\n",
    "            else: \n",
    "                level_agg = pd.concat([level_agg, da_agg], axis=0)\n",
    "                level_agg.fillna(0, inplace=True)\n",
    "                level_agg, _ = reduce_mem_usage(level_agg)\n",
    "        print(level_agg.shape)\n",
    "        level_agg, _ = reduce_mem_usage(level_agg)\n",
    "        dump_pickle(level_agg, feat_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T13:01:22.415628Z",
     "start_time": "2018-04-21T13:01:22.411298Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_level_features():\n",
    "    for c in tqdm(['item_id', 'shop_id', 'user_id']):\n",
    "        gen_level_aggs(c, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T13:01:22.978703Z",
     "start_time": "2018-04-21T13:01:22.967953Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_level_features(data, col):\n",
    "    feat_path = os.path.join(feats_root,'level_aggs_{}.pkl'.format(col))\n",
    "    if not os.path.exists(feat_path):\n",
    "        print('Not found ' + feat_path)\n",
    "        return data\n",
    "    agg = load_pickle(feat_path)\n",
    "    return pd.merge(data, agg, how='left',on=[col, 'da'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T13:01:23.578489Z",
     "start_time": "2018-04-21T13:01:23.564703Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3/3 [00:00<00:00, 1472.72it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found ./feats/level_aggs_item_id.pkl\n",
      "Found ./feats/level_aggs_shop_id.pkl\n",
      "Found ./feats/level_aggs_user_id.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    gen_level_features()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.089266Z",
     "start_time": "2018-04-21T12:23:49.082875Z"
    }
   },
   "outputs": [],
   "source": [
    "# dfal = get_nominal_dfal()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.097389Z",
     "start_time": "2018-04-21T12:23:49.092427Z"
    }
   },
   "outputs": [],
   "source": [
    "# dfal.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.109656Z",
     "start_time": "2018-04-21T12:23:49.102283Z"
    }
   },
   "outputs": [],
   "source": [
    "# dfal = dfal.loc[dfal.da>20,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.119084Z",
     "start_time": "2018-04-21T12:23:49.114616Z"
    }
   },
   "outputs": [],
   "source": [
    "# for c in tqdm_notebook(nominal_cate_cols + ordinal_cate_cols + identity_cols):\n",
    "#     dfal = add_target_features(dfal, c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.130848Z",
     "start_time": "2018-04-21T12:23:49.125234Z"
    }
   },
   "outputs": [],
   "source": [
    "# del dfal['dt']\n",
    "# for c in dfal.columns:\n",
    "#     if c.endswith('_wd_6'):\n",
    "#         del dfal[c]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.139713Z",
     "start_time": "2018-04-21T12:23:49.134462Z"
    }
   },
   "outputs": [],
   "source": [
    "# dfal, _ = reduce_mem_usage(dfal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.148463Z",
     "start_time": "2018-04-21T12:23:49.143876Z"
    }
   },
   "outputs": [],
   "source": [
    "# dfal.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.155940Z",
     "start_time": "2018-04-21T12:23:49.151511Z"
    }
   },
   "outputs": [],
   "source": [
    "# X_tr = dfal.loc[dfal.da<=22,:].drop(['da', 'hm', 'instance_id', 'is_trade'] + identity_cols, axis=1)\n",
    "# y_tr = dfal.loc[dfal.da<=22,'is_trade']\n",
    "# X_va = dfal.loc[dfal.da==23,:].drop(['da', 'hm', 'instance_id', 'is_trade'] + identity_cols, axis=1)\n",
    "# y_va = dfal.loc[dfal.da==23,'is_trade']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.165680Z",
     "start_time": "2018-04-21T12:23:49.159609Z"
    }
   },
   "outputs": [],
   "source": [
    "# %matplotlib inline\n",
    "# import matplotlib.pyplot as plt\n",
    "# import catboost as cb\n",
    "# import xgboost as xg\n",
    "# import lightgbm as lg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.187625Z",
     "start_time": "2018-04-21T12:23:49.172458Z"
    }
   },
   "outputs": [],
   "source": [
    "# def print_feature_importance_lgb(gbm):\n",
    "#     print(80 * '*')\n",
    "#     print(31 * '*' + 'Feature Importance' + 31 * '*')\n",
    "#     print(80 * '.')\n",
    "#     print(\"\\n\".join((\".%50s => %9.5f\" % x) for x in sorted(\n",
    "#         zip(gbm.feature_name(), gbm.feature_importance(\"gain\")),\n",
    "#         key=lambda x: x[1],\n",
    "#         reverse=True)))\n",
    "#     print(80 * '.')\n",
    "\n",
    "# def fit_lgb(X_tr, y_tr, X_va, y_va, cates_cols):\n",
    "#     params = {\n",
    "#         'max_depth': 8,\n",
    "#         'num_leaves': 128,\n",
    "#         'objective':'binary',\n",
    "#         'min_data_in_leaf': 20,\n",
    "#         'learning_rate': 0.01,\n",
    "#         'feature_fraction': 0.9,\n",
    "#         'bagging_fraction': 0.8,\n",
    "#         'subsample':0.85,\n",
    "#         'bagging_freq': 1,\n",
    "#         'random_state':2018,\n",
    "#         'metric': ['binary_logloss'],\n",
    "#         'num_threads': 16,\n",
    "#         #'is_unbalance': True\n",
    "#     }\n",
    "\n",
    "#     MAX_ROUNDS = 10000\n",
    "#     dtr = lg.Dataset(X_tr, label=y_tr, categorical_feature=cates_cols)\n",
    "#     dva = lg.Dataset(X_va, label=y_va, categorical_feature=cates_cols, reference=dtr)\n",
    "    \n",
    "#     cls = lg.train(\n",
    "#         params,\n",
    "#         dtr,\n",
    "#         num_boost_round=MAX_ROUNDS,\n",
    "#         valid_sets=(dva, dtr),\n",
    "#         valid_names=['valid', 'train'],\n",
    "#         early_stopping_rounds=125,\n",
    "#         verbose_eval=50)\n",
    "#     print_feature_importance_lgb(cls)\n",
    "#     lg.plot_importance(cls, importance_type='gain', figsize=(11,12), max_num_features=50, grid=False)\n",
    "#     return cls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.198165Z",
     "start_time": "2018-04-21T12:23:49.191627Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# gbm = fit_lgb(X_tr, y_tr, X_va, y_va, nominal_cate_cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CatBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.207106Z",
     "start_time": "2018-04-21T12:23:49.201733Z"
    }
   },
   "outputs": [],
   "source": [
    "# cates_idx = [X_tr.columns.values.tolist().index(c) for c in nominal_cate_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.237252Z",
     "start_time": "2018-04-21T12:23:49.213004Z"
    }
   },
   "outputs": [],
   "source": [
    "# import operator\n",
    "# def verbose_feature_importance_cat(cls, X_tr):\n",
    "#     cat_feature_importance = {\n",
    "#         X_tr.columns.values.tolist()[idx]: score\n",
    "#         for idx, score in enumerate(cls.feature_importances_)\n",
    "#     }\n",
    "    \n",
    "#     cat_feature_importance = sorted(cat_feature_importance.items(), \n",
    "#                                     key=operator.itemgetter(1), \n",
    "#                                     reverse=False)\n",
    "    \n",
    "#     print(80 * '*')\n",
    "#     print(31 * '*' + 'Feature Importance' + 31 * '*')\n",
    "#     print(80 * '.')\n",
    "#     for feature, score in reversed(cat_feature_importance):\n",
    "#         print(\".%50s => %9.5f\" % (feature, score))\n",
    "#     print(80 * '.')\n",
    "    \n",
    "#     feature_score = pd.DataFrame(cat_feature_importance, columns=['Feature','Score'])\n",
    "    \n",
    "#     plt.rcParams[\"figure.figsize\"] = (11, 12)\n",
    "#     ax = feature_score.tail(50).plot('Feature', 'Score', kind='barh', color='b')\n",
    "#     ax.set_title(\"Catboost Feature Importance Ranking\", fontsize=8)\n",
    "#     ax.set_xlabel('')\n",
    "#     rects = ax.patches\n",
    "#     # get feature score as labels round to 2 decimal\n",
    "#     labels = feature_score.tail(50)['Score'].round(2)\n",
    "#     for rect, label in zip(rects, labels):\n",
    "#         width = rect.get_width()\n",
    "#         ax.text(width + 0.2,rect.get_y()+0.02, label, ha='center', va='bottom')\n",
    "#     plt.show()\n",
    "\n",
    "\n",
    "# def fit_cat(X_tr, y_tr, X_va, y_va, cates_idx):\n",
    "#     print('Fitting CatBoostClassifier ...')\n",
    "#     cls = cb.CatBoostClassifier(\n",
    "#         iterations=2000,\n",
    "#         od_type='Iter',\n",
    "#         od_wait=120,\n",
    "#         max_depth=8,\n",
    "#         learning_rate=0.02,\n",
    "#         l2_leaf_reg=9,\n",
    "#         random_seed=2018,\n",
    "#         metric_period=50,\n",
    "#         fold_len_multiplier=1.1,\n",
    "#         loss_function='Logloss',\n",
    "#         logging_level='Verbose')\n",
    "#     fine_model = cls.fit(X_tr, y_tr, eval_set=(X_va, y_va), cat_features=cates_idx)\n",
    "#     verbose_feature_importance_cat(fine_model, X_tr)\n",
    "#     return fine_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-21T12:23:49.244541Z",
     "start_time": "2018-04-21T12:23:49.240545Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# cat = fit_cat(X_tr, y_tr, X_va, y_va, cates_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "stats36",
   "language": "python",
   "name": "stats36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}