_2_3_gen_shop_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:04.831071Z",
     "start_time": "2018-04-20T09:40:03.816422Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import gc\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from utils import load_pickle,dump_pickle, get_nominal_dfal, feats_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:04.845650Z",
     "start_time": "2018-04-20T09:40:04.833607Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_shop_total_da_click(update=False):\n",
    "    dfal = get_nominal_dfal()\n",
    "    feat_path = os.path.join(feats_root, 'shop_total_click_da.pkl')\n",
    "    if os.path.exists(feat_path) and update == False:\n",
    "        print('Found ' + feat_path)\n",
    "    else:\n",
    "        print('Generating ' + feat_path)\n",
    "        shop_all_click_da = dfal.groupby(['shop_id', 'da'])\\\n",
    "                                 .size().reset_index()\\\n",
    "                                 .rename(columns={0: 'agg_shop_total_click_da'})\n",
    "        dump_pickle(shop_all_click_da, feat_path)\n",
    "        \n",
    "    print('gen_shop_total_da_click completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:04.874013Z",
     "start_time": "2018-04-20T09:40:04.848599Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_shop_da_feature_click(updata=False):\n",
    "    \"\"\"生成用户相关所有数据的每天点击统计量\"\"\"\n",
    "    dfal = get_nominal_dfal()\n",
    "    stats_feat = [\n",
    "        'item_category_list', 'item_brand_id', 'item_city_id',\n",
    "        'user_gender_id', 'user_occupation_id', 'item_price_level',\n",
    "        'item_sales_level', 'item_collected_level', 'item_pv_level',\n",
    "        'user_age_level', 'user_star_level', 'context_page_id', 'item_id',\n",
    "        'user_id'\n",
    "    ]\n",
    "    tbar = tqdm(stats_feat)\n",
    "    for feat in tbar:\n",
    "        feat_path = os.path.join(feats_root, 'shop_' + feat + '_click_da.pkl')\n",
    "        if os.path.exists(feat_path) and updata == False:\n",
    "            tbar.set_description('Found {:>60}'.format(os.path.basename(feat_path)))\n",
    "        else:\n",
    "            tbar.set_description('Generating {:>60}'.format(\n",
    "                os.path.basename(feat_path)))\n",
    "            shop_feat_click_da = dfal.groupby(['shop_id', 'da', feat])\\\n",
    "                                     .size().reset_index()\\\n",
    "                                     .rename(columns={0: 'agg_shop_%s_click_da' % feat})\n",
    "            dump_pickle(shop_feat_click_da, feat_path)\n",
    "\n",
    "    print('gen_shop_da_feature_click completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:04.900805Z",
     "start_time": "2018-04-20T09:40:04.877855Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_shop_ho_feature_click(updata=False):\n",
    "    \"\"\"生成用户相关所有数据的每天每小时点击统计量\"\"\"\n",
    "    dfal = get_nominal_dfal()\n",
    "    stats_feat = [\n",
    "        'item_category_list', 'item_brand_id', 'item_city_id',\n",
    "        'user_gender_id', 'user_occupation_id', 'item_price_level',\n",
    "        'item_sales_level', 'item_collected_level', 'item_pv_level',\n",
    "        'user_age_level', 'user_star_level', 'context_page_id', 'item_id',\n",
    "        'user_id'\n",
    "    ]\n",
    "    tbar = tqdm(stats_feat)\n",
    "    for feat in tbar:\n",
    "        feat_path = os.path.join(feats_root, 'shop_' + feat + '_click_ho.pkl')\n",
    "        if os.path.exists(feat_path) and updata == False:\n",
    "            tbar.set_description('Found {:>60}'.format(os.path.basename(feat_path)))\n",
    "        else:\n",
    "            tbar.set_description('Generating {:>60}'.format(os.path.basename(feat_path)))\n",
    "            shop_feat_click_ho = dfal.groupby(['shop_id', 'da', 'ho', feat])\\\n",
    "                                     .size().reset_index()\\\n",
    "                                     .rename(columns={0: 'agg_shop_%s_click_ho' % feat})\n",
    "            dump_pickle(shop_feat_click_ho, feat_path)\n",
    "    print('gen_shop_ho_feature_click completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:04.915368Z",
     "start_time": "2018-04-20T09:40:04.905502Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_shop_total_da_click(data):\n",
    "    \"\"\"\n",
    "    添加用户当天的点击总数\n",
    "    拼接键['user_id', 'da']\n",
    "    \"\"\"\n",
    "    feat_path = feats_root + 'shop_total_click_da.pkl'\n",
    "    if not os.path.exists(feat_path):\n",
    "        gen_shop_total_da_click()\n",
    "    shop_total_click_da = load_pickle(feat_path)\n",
    "    data = pd.merge(data, shop_total_click_da, 'left', ['da','shop_id'])\n",
    "    print('add_shop_total_da_click completed.')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:04.936076Z",
     "start_time": "2018-04-20T09:40:04.918892Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_shop_da_feature_click(data):\n",
    "    stats_feat = [\n",
    "        'item_category_list', 'item_brand_id', 'item_city_id',\n",
    "        'user_gender_id', 'user_occupation_id', 'item_price_level',\n",
    "        'item_sales_level', 'item_collected_level', 'item_pv_level',\n",
    "        'user_age_level', 'user_star_level', 'context_page_id', 'item_id',\n",
    "        'user_id'\n",
    "    ]\n",
    "    \n",
    "    tbar = tqdm(stats_feat)\n",
    "    for feat in tbar:\n",
    "        feat_path = os.path.join(feats_root, 'shop_' + feat + '_click_da.pkl')\n",
    "        feat_da_click = load_pickle(feat_path)\n",
    "        tbar.set_description('adding ' + os.path.basename(feat_path))\n",
    "        data = pd.merge(data, feat_da_click, 'left', [feat, 'da', 'shop_id'])\n",
    "    print('add_shop_da_feature_click completed.')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:04.962649Z",
     "start_time": "2018-04-20T09:40:04.941151Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_shop_ho_feature_click(data):\n",
    "    stats_feat = [\n",
    "        'item_category_list', 'item_brand_id', 'item_city_id',\n",
    "        'user_gender_id', 'user_occupation_id', 'item_price_level',\n",
    "        'item_sales_level', 'item_collected_level', 'item_pv_level',\n",
    "        'user_age_level', 'user_star_level', 'context_page_id', 'item_id',\n",
    "        'user_id'\n",
    "    ]\n",
    "    \n",
    "    tbar = tqdm(stats_feat)\n",
    "    for feat in tbar:\n",
    "        feat_path = os.path.join(feats_root, 'shop_' + feat + '_click_ho.pkl')\n",
    "        feat_da_click = load_pickle(feat_path)\n",
    "        tbar.set_description('adding ' + os.path.basename(feat_path))\n",
    "        data = pd.merge(data, feat_da_click, 'left', [feat, 'ho', 'da', 'shop_id'])\n",
    "    print('add_shop_ho_feature_click completed.') \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:04.994285Z",
     "start_time": "2018-04-20T09:40:04.967052Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_shop_click_stats(data, col):\n",
    "    clicks_shop = pd.DataFrame(data.groupby(['shop_id', col])['dt'].count(), )\n",
    "    clicks_shop.rename(columns={'dt': col+'_m'}, inplace=True)\n",
    "    clicks_shop.reset_index(inplace=True)\n",
    "    clicks_shop_avg = pd.DataFrame(clicks_shop.groupby(['shop_id'])[col+'_m'].mean()).rename(columns={col+'_m': col+'_avg'}).reset_index()\n",
    "    clicks_shop_max = pd.DataFrame(clicks_shop.groupby(['shop_id'])[col+'_m'].max()).rename(columns={col+'_m': col+'_max'}).reset_index()\n",
    "    clicks_shop_min = pd.DataFrame(clicks_shop.groupby(['shop_id'])[col+'_m'].min()).rename(columns={col+'_m': col+'_min'}).reset_index()\n",
    "    data = pd.merge(data, clicks_shop_avg, how='left', on='shop_id')\n",
    "    data = pd.merge(data, clicks_shop_max, how='left', on='shop_id')\n",
    "    data = pd.merge(data, clicks_shop_min, how='left', on='shop_id')\n",
    "    print('gen_shop_click_stats {} completed.'.format(col))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:05.006339Z",
     "start_time": "2018-04-20T09:40:04.997048Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_shop_click_stats(data):\n",
    "    feat_path = os.path.join(feats_root, 'shop_click_stats.pkl')\n",
    "    if not os.path.exists(feat_path):\n",
    "        gen_shop_stats_feature()\n",
    "    shop_click_stats = load_pickle(feat_path)\n",
    "    data = pd.merge(data, shop_click_stats, how='left', on='shop_id')\n",
    "    print('add_shop_click_stats completed.')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:05.050089Z",
     "start_time": "2018-04-20T09:40:05.009625Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_shop_stats_feature(updata=False):\n",
    "    feat_path = os.path.join(feats_root, 'shop_click_stats.pkl')\n",
    "    if os.path.exists(feat_path) and updata == False:\n",
    "        print('Found ' + feat_path)\n",
    "    else:\n",
    "        dfal = get_nominal_dfal()\n",
    "        dfal = add_shop_total_da_click(dfal)\n",
    "        dfal = add_shop_da_feature_click(dfal)\n",
    "        print('generating ' + feat_path)\n",
    "        columns_da = list(filter(lambda x: x.endswith('_click_da'), dfal.columns.values))\n",
    "        columns_ho = list(filter(lambda x: x.endswith('_click_ho'), dfal.columns.values))\n",
    "\n",
    "        tbar = tqdm(columns_da)\n",
    "        for col in tbar:\n",
    "            tbar.set_description('add_shop_click_stats ' + col)\n",
    "            dfal = gen_shop_click_stats(dfal, col)\n",
    "        print('gen_shop_click_stats completed.')\n",
    "        \n",
    "        feat_names = list(filter(lambda x: '_click_da_' in x, dfal.columns.values))\n",
    "        dfal = dfal[feat_names + ['shop_id']].drop_duplicates(['shop_id'])\n",
    "        print('gen_shop_stats_feature shape:', dfal.shape)\n",
    "        dump_pickle(dfal, feat_path)\n",
    "    print('gen_shop_stats_feature completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:05.280340Z",
     "start_time": "2018-04-20T09:40:05.052933Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found                                    shop_user_id_click_da.pkl: 100%|██████████| 14/14 [00:00<00:00, 453.40it/s]\n",
      "Found                                    shop_user_id_click_ho.pkl: 100%|██████████| 14/14 [00:00<00:00, 343.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found ./feats/shop_total_click_da.pkl\n",
      "gen_shop_total_da_click completed.\n",
      "gen_shop_da_feature_click completed.\n",
      "gen_shop_ho_feature_click completed.\n",
      "Found ./feats/shop_click_stats.pkl\n",
      "gen_shop_stats_feature completed.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    gen_shop_total_da_click(False)\n",
    "    gen_shop_da_feature_click(False)\n",
    "    gen_shop_ho_feature_click(False)\n",
    "    gen_shop_stats_feature(False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "stats36",
   "language": "python",
   "name": "stats36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}