_2_2_gen_item_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.357306Z",
     "start_time": "2018-04-20T09:40:51.421738Z"
    },
    "cell_style": "center"
   },
   "outputs": [],
   "source": [
    "import gc, os, pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from utils import load_pickle, dump_pickle, get_nominal_dfal, feats_root"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.371827Z",
     "start_time": "2018-04-20T09:40:52.360230Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_item_total_da_click(update=False):\n",
    "    dfal = get_nominal_dfal()\n",
    "    feat_path = os.path.join(feats_root, 'item_total_click_da.pkl')\n",
    "    if os.path.exists(feat_path) and update == False:\n",
    "        print('Found ' + feat_path)\n",
    "    else:\n",
    "        print('Generating ' + feat_path)\n",
    "        item_all_click_da = dfal.groupby(['item_id', 'da'])\\\n",
    "                                 .size().reset_index()\\\n",
    "                                 .rename(columns={0: 'agg_item_total_click_da'})\n",
    "        dump_pickle(item_all_click_da, feat_path)\n",
    "        \n",
    "    print('gen_item_total_da_click completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.399503Z",
     "start_time": "2018-04-20T09:40:52.374709Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_item_da_feature_click(updata=False):\n",
    "    \"\"\"生成用户相关所有数据的每天点击统计量\"\"\"\n",
    "    dfal = get_nominal_dfal()\n",
    "    stats_feat = [\n",
    "        'shop_id', 'user_id', 'user_gender_id', 'user_occupation_id',\n",
    "        'user_age_level', 'user_star_level', 'context_page_id',\n",
    "        'shop_review_num_level', 'shop_star_level'\n",
    "    ]\n",
    "    tbar = tqdm(stats_feat)\n",
    "    for feat in tbar:\n",
    "        feat_path = os.path.join(feats_root, 'item_' + feat + '_click_da.pkl')\n",
    "        if os.path.exists(feat_path) and updata == False:\n",
    "            tbar.set_description('Found {:>60}'.format(\n",
    "                os.path.basename(feat_path)))\n",
    "        else:\n",
    "            tbar.set_description('Generating {:>60}'.format(\n",
    "                os.path.basename(feat_path)))\n",
    "            item_feat_click_da = dfal.groupby(['item_id', 'da', feat])\\\n",
    "                                     .size().reset_index()\\\n",
    "                                     .rename(columns={0: 'agg_item_%s_click_da' % feat})\n",
    "            dump_pickle(item_feat_click_da, feat_path)\n",
    "\n",
    "    print('gen_item_da_feature_click completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.426896Z",
     "start_time": "2018-04-20T09:40:52.401870Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_item_ho_feature_click(updata=False):\n",
    "    \"\"\"生成用户相关所有数据的每天每小时点击统计量\"\"\"\n",
    "    dfal = get_nominal_dfal()\n",
    "    stats_feat = [\n",
    "        'shop_id', 'user_id', 'user_gender_id', 'user_occupation_id',\n",
    "        'user_age_level', 'user_star_level', 'context_page_id',\n",
    "        'shop_review_num_level', 'shop_star_level'\n",
    "    ]\n",
    "    tbar = tqdm(stats_feat)\n",
    "    for feat in tbar:\n",
    "        feat_path = os.path.join(feats_root, 'item_' + feat + '_click_ho.pkl')\n",
    "        if os.path.exists(feat_path) and updata == False:\n",
    "            tbar.set_description('Found {:>60}'.format(os.path.basename(feat_path)))\n",
    "        else:\n",
    "            tbar.set_description('Generating {:>60}'.format(os.path.basename(feat_path)))\n",
    "            item_feat_click_ho = dfal.groupby(['item_id', 'da', 'ho', feat])\\\n",
    "                                     .size().reset_index()\\\n",
    "                                     .rename(columns={0: 'agg_item_%s_click_ho' % feat})\n",
    "            dump_pickle(item_feat_click_ho, feat_path)\n",
    "    print('gen_item_ho_feature_click completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.438633Z",
     "start_time": "2018-04-20T09:40:52.430366Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_item_total_da_click(data):\n",
    "    \"\"\"\n",
    "    添加用户当天的点击总数\n",
    "    拼接键['item_id', 'da']\n",
    "    \"\"\"\n",
    "    feat_path = feats_root + 'item_total_click_da.pkl'\n",
    "    if not os.path.exists(feat_path):\n",
    "        gen_item_total_da_click()\n",
    "    item_total_click_da = load_pickle(feat_path)\n",
    "    data = pd.merge(data, item_total_click_da, 'left', ['da','item_id'])\n",
    "    print('add_item_total_da_click completed.')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.459856Z",
     "start_time": "2018-04-20T09:40:52.441609Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_item_da_feature_click(data):\n",
    "    stats_feat = [\n",
    "        'shop_id', 'user_id', 'user_gender_id', 'user_occupation_id',\n",
    "        'user_age_level', 'user_star_level', 'context_page_id',\n",
    "        'shop_review_num_level', 'shop_star_level'\n",
    "    ]\n",
    "    tbar = tqdm(stats_feat)\n",
    "    for feat in tbar:\n",
    "        feat_path = os.path.join(feats_root, 'item_' + feat + '_click_da.pkl')\n",
    "        feat_da_click = load_pickle(feat_path)\n",
    "        tbar.set_description('adding ' + os.path.basename(feat_path))\n",
    "        data = pd.merge(data, feat_da_click, 'left', [feat, 'da', 'item_id'])\n",
    "    print('add_item_da_feature_click completed.')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.476778Z",
     "start_time": "2018-04-20T09:40:52.462809Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_item_ho_feature_click(data):\n",
    "    stats_feat = [\n",
    "        'shop_id', 'user_id', 'user_gender_id', 'user_occupation_id',\n",
    "        'user_age_level', 'user_star_level', 'context_page_id',\n",
    "        'shop_review_num_level', 'shop_star_level'\n",
    "    ]\n",
    "    \n",
    "    tbar = tqdm(stats_feat)\n",
    "    for feat in tbar:\n",
    "        feat_path = os.path.join(feats_root, 'item_' + feat + '_click_ho.pkl')\n",
    "        feat_da_click = load_pickle(feat_path)\n",
    "        tbar.set_description('adding ' + os.path.basename(feat_path))\n",
    "        data = pd.merge(data, feat_da_click, 'left', [feat, 'ho', 'da', 'item_id'])\n",
    "    print('add_item_ho_feature_click completed.') \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.507636Z",
     "start_time": "2018-04-20T09:40:52.480026Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_item_click_stats(data, col):\n",
    "    clicks_item = pd.DataFrame(data.groupby(['item_id', col])['dt'].count(), )\n",
    "    clicks_item.rename(columns={'dt': col+'_m'}, inplace=True)\n",
    "    clicks_item.reset_index(inplace=True)\n",
    "    clicks_item_avg = pd.DataFrame(clicks_item.groupby(['item_id'])[col+'_m'].mean()).rename(columns={col+'_m': col+'_avg'}).reset_index()\n",
    "    clicks_item_max = pd.DataFrame(clicks_item.groupby(['item_id'])[col+'_m'].max()).rename(columns={col+'_m': col+'_max'}).reset_index()\n",
    "    clicks_item_min = pd.DataFrame(clicks_item.groupby(['item_id'])[col+'_m'].min()).rename(columns={col+'_m': col+'_min'}).reset_index()\n",
    "    data = pd.merge(data, clicks_item_avg, how='left', on='item_id')\n",
    "    data = pd.merge(data, clicks_item_max, how='left', on='item_id')\n",
    "    data = pd.merge(data, clicks_item_min, how='left', on='item_id')\n",
    "    print('add_item_click_stats {} completed.'.format(col))\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.524840Z",
     "start_time": "2018-04-20T09:40:52.510825Z"
    }
   },
   "outputs": [],
   "source": [
    "def add_item_click_stats(data):\n",
    "    feat_path = os.path.join(feats_root, 'item_click_stats.pkl')\n",
    "    if not os.path.exists(feat_path):\n",
    "        gen_item_stats_feature()\n",
    "    item_click_stats = load_pickle(feat_path)\n",
    "    data = pd.merge(data, item_click_stats, how='left', on='item_id')\n",
    "    print('add_item_click_stats completed.')\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.568652Z",
     "start_time": "2018-04-20T09:40:52.527586Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_item_stats_feature(updata=False):\n",
    "    feat_path = os.path.join(feats_root, 'item_click_stats.pkl')\n",
    "    if os.path.exists(feat_path) and updata == False:\n",
    "        print('Found ' + feat_path)\n",
    "    else:\n",
    "        dfal = get_nominal_dfal()\n",
    "        dfal = add_item_total_da_click(dfal)\n",
    "        dfal = add_item_da_feature_click(dfal)\n",
    "        print('generating ' + feat_path)\n",
    "        columns_da = list(filter(lambda x: x.endswith('_click_da'), dfal.columns.values))\n",
    "        columns_ho = list(filter(lambda x: x.endswith('_click_ho'), dfal.columns.values))\n",
    "\n",
    "        tbar = tqdm(columns_da)\n",
    "        for col in tbar:\n",
    "            tbar.set_description('add_item_click_stats ' + col)\n",
    "            dfal = gen_item_click_stats(dfal, col)\n",
    "        print('add_item_click_stats completed.')\n",
    "        \n",
    "        feat_names = list(filter(lambda x: '_click_da_' in x, dfal.columns.values))\n",
    "        dfal = dfal[feat_names + ['item_id']].drop_duplicates(['item_id'])\n",
    "        print('gen_item_stats_feature shape:', dfal.shape)\n",
    "        dump_pickle(dfal, feat_path)\n",
    "    print('gen_item_stats_feature completed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-20T09:40:52.786440Z",
     "start_time": "2018-04-20T09:40:52.571374Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found                            item_shop_star_level_click_da.pkl: 100%|██████████| 9/9 [00:00<00:00, 340.91it/s]\n",
      "Found                            item_shop_star_level_click_ho.pkl: 100%|██████████| 9/9 [00:00<00:00, 285.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found ./feats/item_total_click_da.pkl\n",
      "gen_item_total_da_click completed.\n",
      "gen_item_da_feature_click completed.\n",
      "gen_item_ho_feature_click completed.\n",
      "Found ./feats/item_click_stats.pkl\n",
      "gen_item_stats_feature completed.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    gen_item_total_da_click(False)\n",
    "    gen_item_da_feature_click(False)\n",
    "    gen_item_ho_feature_click(False)\n",
    "    gen_item_stats_feature(False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "stats36",
   "language": "python",
   "name": "stats36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}