_2_5_gen_smooth_cvr.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:44:39.836909Z",
     "start_time": "2018-04-20T09:44:39.823903Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import gc\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "np.random.seed(2018)\n",
    "import random\n",
    "import scipy.special as special\n",
    "\n",
    "from tqdm import tqdm\n",
    "from utils import (load_pickle, dump_pickle, get_nominal_dfal, feats_root,\n",
    "                   ordinal_cate_cols, nominal_cate_cols, identity_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:44:41.593679Z",
     "start_time": "2018-04-20T09:44:41.525770Z"
    },
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "class BayesianSmoothing(object):\n",
    "    def __init__(self, alpha, beta):\n",
    "        self.alpha = alpha\n",
    "        self.beta = beta\n",
    "\n",
    "    def sample(self, alpha, beta, num, imp_upperbound):\n",
    "        sample = np.random.beta(alpha, beta, num)\n",
    "        I = []\n",
    "        C = []\n",
    "        for clk_rt in sample:\n",
    "            imp = random.random() * imp_upperbound\n",
    "            imp = imp_upperbound\n",
    "            clk = imp * clk_rt\n",
    "            I.append(imp)\n",
    "            C.append(clk)\n",
    "        return I, C\n",
    "\n",
    "    def update(self, imps, clks, iter_num, epsilon):\n",
    "        for i in tqdm_notebook(range(iter_num)):\n",
    "            new_alpha, new_beta = self.__fixed_point_iteration(\n",
    "                imps, clks, self.alpha, self.beta)\n",
    "            if abs(new_alpha - self.alpha) < epsilon and abs(\n",
    "                    new_beta - self.beta) < epsilon:\n",
    "                break\n",
    "            #print (new_alpha,new_beta,i)\n",
    "            self.alpha = new_alpha\n",
    "            self.beta = new_beta\n",
    "\n",
    "    def __fixed_point_iteration(self, imps, clks, alpha, beta):\n",
    "        numerator_alpha = 0.0\n",
    "        numerator_beta = 0.0\n",
    "        denominator = 0.0\n",
    "\n",
    "        for i in range(len(imps)):\n",
    "            numerator_alpha += (\n",
    "                special.digamma(clks[i] + alpha) - special.digamma(alpha))\n",
    "            numerator_beta += (special.digamma(imps[i] - clks[i] + beta) -\n",
    "                               special.digamma(beta))\n",
    "            denominator += (special.digamma(imps[i] + alpha + beta) -\n",
    "                            special.digamma(alpha + beta))\n",
    "\n",
    "        return alpha * (numerator_alpha / denominator), beta * (\n",
    "            numerator_beta / denominator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:44:42.007307Z",
     "start_time": "2018-04-20T09:44:41.975095Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_hist_cvr_smooth(start_da, end_da, key, alpha=0.25):\n",
    "    dfal = get_nominal_dfal()\n",
    "    dfal = dfal.loc[dfal.da <= end_da, [key, 'da', 'is_trade']]\n",
    "    gc.collect()\n",
    "    for da in tqdm(np.arange(start_da, end_da + 1)):\n",
    "        feat_path = os.path.join(\n",
    "            feats_root, key + '_hist_cvr_smooth_da_' + str(da) + '.pkl')\n",
    "        if os.path.exists(feat_path):\n",
    "            print('found ' + feat_path)\n",
    "        else:\n",
    "            print('generating ' + feat_path)\n",
    "            dfcv =  dfal.copy().loc[dfal.da < da]\n",
    "            dfcv.is_trade = dfcv.is_trade.apply(int)\n",
    "            dfcv = pd.get_dummies(dfcv, columns=['is_trade'], prefix='label')\n",
    "            dfcv = dfcv.groupby([key], as_index=False).sum()\n",
    "            dfcv[key + '_cvr'] = (dfcv['label_1'] + alpha) / (dfcv['label_0'] + dfcv['label_1'] + alpha * 2)\n",
    "            result = pd.merge(\n",
    "                dfal.loc[dfal.da == da, ['da', key]],\n",
    "                dfcv.loc[:, [key, key + '_cvr']],\n",
    "                'left',\n",
    "                on=[key,])\n",
    "            result.drop_duplicates(['da', key], inplace=True)\n",
    "            result.sort_values(['da', key], inplace=True)\n",
    "            dump_pickle(result.loc[:, ['da', key, key + '_cvr']], feat_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:44:43.115888Z",
     "start_time": "2018-04-20T09:44:43.101141Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_hist_cvr_smooth(data, key):\n",
    "    hist_cvr_smooth = None\n",
    "    tbar = tqdm(sorted(data.da.unique()))\n",
    "    for da in tbar:\n",
    "        feat_path = os.path.join(feats_root, key + '_hist_cvr_smooth_da_' + str(da) + '.pkl')\n",
    "        tbar.set_description('adding hist cvr smooth {},{}'.format(key, da))\n",
    "        da_cvr_smooth = load_pickle(feat_path)\n",
    "        if hist_cvr_smooth is None:\n",
    "            hist_cvr_smooth = da_cvr_smooth\n",
    "        else:\n",
    "            hist_cvr_smooth = pd.concat([hist_cvr_smooth, da_cvr_smooth], axis=0)\n",
    "    data = pd.merge(data, hist_cvr_smooth, 'left', ['da', key])\n",
    "    print('add_hist_cvr_smooth {} completed'.format(key))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:46:13.485220Z",
     "start_time": "2018-04-20T09:45:49.144380Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/20 [00:00<?, ?it/s]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  7.42it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_price_level_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/item_price_level_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  6.59it/s]\u001b[A\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.06it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_price_level_hist_cvr_smooth_da_20.pkl\n",
      "generating ./feats/item_price_level_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_price_level_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:01<00:00,  4.95it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_price_level_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_price_level_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.14it/s]\u001b[A\n",
      "  5%|▌         | 1/20 [00:01<00:35,  1.89s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  7.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_sales_level_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/item_sales_level_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  6.82it/s]\u001b[A\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.14it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_sales_level_hist_cvr_smooth_da_20.pkl\n",
      "generating ./feats/item_sales_level_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.19it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_sales_level_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:01<00:00,  4.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_sales_level_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_sales_level_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.09it/s]\u001b[A\n",
      " 10%|█         | 2/20 [00:03<00:34,  1.89s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  7.29it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_collected_level_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/item_collected_level_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  6.46it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_collected_level_hist_cvr_smooth_da_20.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  5.81it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_collected_level_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.33it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_collected_level_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:01<00:00,  4.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_collected_level_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.35it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_collected_level_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.03it/s]\u001b[A\n",
      " 15%|█▌        | 3/20 [00:05<00:32,  1.91s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  7.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_pv_level_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/item_pv_level_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.18it/s]\u001b[A\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.53it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_pv_level_hist_cvr_smooth_da_20.pkl\n",
      "generating ./feats/item_pv_level_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_pv_level_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:01<00:00,  4.56it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_pv_level_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  3.99it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/item_pv_level_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  3.81it/s]\u001b[A\n",
      " 20%|██        | 4/20 [00:07<00:31,  1.95s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  8.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_age_level_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/user_age_level_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.29it/s]\u001b[A\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.58it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_age_level_hist_cvr_smooth_da_20.pkl\n",
      "generating ./feats/user_age_level_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_age_level_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:00<00:00,  5.36it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_age_level_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_age_level_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.53it/s]\u001b[A\n",
      " 25%|██▌       | 5/20 [00:09<00:28,  1.90s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  8.54it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_star_level_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/user_star_level_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.41it/s]\u001b[A\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_star_level_hist_cvr_smooth_da_20.pkl\n",
      "generating ./feats/user_star_level_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_star_level_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:00<00:00,  5.39it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_star_level_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.92it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_star_level_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.51it/s]\u001b[A\n",
      " 30%|███       | 6/20 [00:11<00:26,  1.87s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 35%|███▌      | 7/20 [00:11<00:21,  1.63s/it]]\u001b[A\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found ./feats/context_page_id_hist_cvr_smooth_da_18.pkl\n",
      "found ./feats/context_page_id_hist_cvr_smooth_da_19.pkl\n",
      "found ./feats/context_page_id_hist_cvr_smooth_da_20.pkl\n",
      "found ./feats/context_page_id_hist_cvr_smooth_da_21.pkl\n",
      "found ./feats/context_page_id_hist_cvr_smooth_da_22.pkl\n",
      "found ./feats/context_page_id_hist_cvr_smooth_da_23.pkl\n",
      "found ./feats/context_page_id_hist_cvr_smooth_da_24.pkl\n",
      "generating ./feats/shop_review_num_level_hist_cvr_smooth_da_18.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  8.87it/s]\u001b[A\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.88it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_review_num_level_hist_cvr_smooth_da_19.pkl\n",
      "generating ./feats/shop_review_num_level_hist_cvr_smooth_da_20.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_review_num_level_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_review_num_level_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:00<00:00,  5.41it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_review_num_level_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_review_num_level_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.53it/s]\u001b[A\n",
      " 40%|████      | 8/20 [00:13<00:19,  1.64s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  9.09it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_star_level_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/shop_star_level_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.81it/s]\u001b[A\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.80it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_star_level_hist_cvr_smooth_da_20.pkl\n",
      "generating ./feats/shop_star_level_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  6.02it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_star_level_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:00<00:00,  5.44it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_star_level_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.96it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/shop_star_level_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.58it/s]\u001b[A\n",
      " 45%|████▌     | 9/20 [00:14<00:18,  1.64s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 50%|█████     | 10/20 [00:14<00:14,  1.49s/it]\u001b[A\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 55%|█████▌    | 11/20 [00:15<00:12,  1.37s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found ./feats/item_category_list_hist_cvr_smooth_da_18.pkl\n",
      "found ./feats/item_category_list_hist_cvr_smooth_da_19.pkl\n",
      "found ./feats/item_category_list_hist_cvr_smooth_da_20.pkl\n",
      "found ./feats/item_category_list_hist_cvr_smooth_da_21.pkl\n",
      "found ./feats/item_category_list_hist_cvr_smooth_da_22.pkl\n",
      "found ./feats/item_category_list_hist_cvr_smooth_da_23.pkl\n",
      "found ./feats/item_category_list_hist_cvr_smooth_da_24.pkl\n",
      "found ./feats/item_brand_id_hist_cvr_smooth_da_18.pkl\n",
      "found ./feats/item_brand_id_hist_cvr_smooth_da_19.pkl\n",
      "found ./feats/item_brand_id_hist_cvr_smooth_da_20.pkl\n",
      "found ./feats/item_brand_id_hist_cvr_smooth_da_21.pkl\n",
      "found ./feats/item_brand_id_hist_cvr_smooth_da_22.pkl\n",
      "found ./feats/item_brand_id_hist_cvr_smooth_da_23.pkl\n",
      "found ./feats/item_brand_id_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 60%|██████    | 12/20 [00:15<00:10,  1.27s/it]\u001b[A\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found ./feats/item_city_id_hist_cvr_smooth_da_18.pkl\n",
      "found ./feats/item_city_id_hist_cvr_smooth_da_19.pkl\n",
      "found ./feats/item_city_id_hist_cvr_smooth_da_20.pkl\n",
      "found ./feats/item_city_id_hist_cvr_smooth_da_21.pkl\n",
      "found ./feats/item_city_id_hist_cvr_smooth_da_22.pkl\n",
      "found ./feats/item_city_id_hist_cvr_smooth_da_23.pkl\n",
      "found ./feats/item_city_id_hist_cvr_smooth_da_24.pkl\n",
      "generating ./feats/user_gender_id_hist_cvr_smooth_da_18.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  8.71it/s]\u001b[A\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.79it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_gender_id_hist_cvr_smooth_da_19.pkl\n",
      "generating ./feats/user_gender_id_hist_cvr_smooth_da_20.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.68it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_gender_id_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.91it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_gender_id_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:00<00:00,  5.30it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_gender_id_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.85it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_gender_id_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.52it/s]\u001b[A\n",
      " 65%|██████▌   | 13/20 [00:16<00:09,  1.30s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  8.77it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_occupation_id_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/user_occupation_id_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.83it/s]\u001b[A\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.87it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_occupation_id_hist_cvr_smooth_da_20.pkl\n",
      "generating ./feats/user_occupation_id_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  6.03it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_occupation_id_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:00<00:00,  5.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_occupation_id_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.89it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/user_occupation_id_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.42it/s]\u001b[A\n",
      " 70%|███████   | 14/20 [00:18<00:08,  1.33s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 75%|███████▌  | 15/20 [00:18<00:06,  1.26s/it]\u001b[A\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 80%|████████  | 16/20 [00:19<00:04,  1.19s/it]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found ./feats/item_id_hist_cvr_smooth_da_18.pkl\n",
      "found ./feats/item_id_hist_cvr_smooth_da_19.pkl\n",
      "found ./feats/item_id_hist_cvr_smooth_da_20.pkl\n",
      "found ./feats/item_id_hist_cvr_smooth_da_21.pkl\n",
      "found ./feats/item_id_hist_cvr_smooth_da_22.pkl\n",
      "found ./feats/item_id_hist_cvr_smooth_da_23.pkl\n",
      "found ./feats/item_id_hist_cvr_smooth_da_24.pkl\n",
      "found ./feats/shop_id_hist_cvr_smooth_da_18.pkl\n",
      "found ./feats/shop_id_hist_cvr_smooth_da_19.pkl\n",
      "found ./feats/shop_id_hist_cvr_smooth_da_20.pkl\n",
      "found ./feats/shop_id_hist_cvr_smooth_da_21.pkl\n",
      "found ./feats/shop_id_hist_cvr_smooth_da_22.pkl\n",
      "found ./feats/shop_id_hist_cvr_smooth_da_23.pkl\n",
      "found ./feats/shop_id_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 85%|████████▌ | 17/20 [00:19<00:03,  1.13s/it]\u001b[A\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "found ./feats/user_id_hist_cvr_smooth_da_18.pkl\n",
      "found ./feats/user_id_hist_cvr_smooth_da_19.pkl\n",
      "found ./feats/user_id_hist_cvr_smooth_da_20.pkl\n",
      "found ./feats/user_id_hist_cvr_smooth_da_21.pkl\n",
      "found ./feats/user_id_hist_cvr_smooth_da_22.pkl\n",
      "found ./feats/user_id_hist_cvr_smooth_da_23.pkl\n",
      "found ./feats/user_id_hist_cvr_smooth_da_24.pkl\n",
      "generating ./feats/hm_hist_cvr_smooth_da_18.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  8.21it/s]\u001b[A\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.24it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/hm_hist_cvr_smooth_da_19.pkl\n",
      "generating ./feats/hm_hist_cvr_smooth_da_20.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.37it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/hm_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.74it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/hm_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:00<00:00,  5.18it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/hm_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.75it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/hm_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.38it/s]\u001b[A\n",
      " 90%|█████████ | 18/20 [00:20<00:02,  1.16s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  8.84it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/mi_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/mi_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.50it/s]\u001b[A\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.71it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/mi_hist_cvr_smooth_da_20.pkl\n",
      "generating ./feats/mi_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  5.97it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/mi_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:00<00:00,  5.43it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/mi_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.93it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/mi_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.57it/s]\u001b[A\n",
      " 95%|█████████▌| 19/20 [00:22<00:01,  1.19s/it]\n",
      "  0%|          | 0/7 [00:00<?, ?it/s]\u001b[A\n",
      " 14%|█▍        | 1/7 [00:00<00:00,  8.72it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/ho_hist_cvr_smooth_da_18.pkl\n",
      "generating ./feats/ho_hist_cvr_smooth_da_19.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 29%|██▊       | 2/7 [00:00<00:00,  7.74it/s]\u001b[A\n",
      " 43%|████▎     | 3/7 [00:00<00:00,  6.90it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/ho_hist_cvr_smooth_da_20.pkl\n",
      "generating ./feats/ho_hist_cvr_smooth_da_21.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 57%|█████▋    | 4/7 [00:00<00:00,  6.10it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/ho_hist_cvr_smooth_da_22.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 71%|███████▏  | 5/7 [00:00<00:00,  5.49it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/ho_hist_cvr_smooth_da_23.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      " 86%|████████▌ | 6/7 [00:01<00:00,  4.94it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generating ./feats/ho_hist_cvr_smooth_da_24.pkl\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 7/7 [00:01<00:00,  4.54it/s]\u001b[A\n",
      "100%|██████████| 20/20 [00:24<00:00,  1.22s/it]\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    for c in tqdm(ordinal_cate_cols + nominal_cate_cols + identity_cols + ['hm','mi','ho'] ):\n",
    "        gen_hist_cvr_smooth(18, 24, c)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "stats36",
   "language": "python",
   "name": "stats36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}