_2_8_gen_last_features.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成用户上一次访问相关的特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-22T02:07:05.984769Z",
     "start_time": "2018-04-22T02:07:03.065565Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import gc\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from utils import (load_pickle, dump_pickle, get_nominal_dfal, feats_root,\n",
    "                   mem_usage, reduce_mem_usage, nominal_cate_cols,\n",
    "                   ordinal_cate_cols, identity_cols, continual_cols, top, freq,\n",
    "                   unique, vrange, percentile)\n",
    "\n",
    "pd.set_option('display.max_columns', 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-22T02:08:22.919659Z",
     "start_time": "2018-04-22T02:08:22.778971Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_user_time_delta_feature(updata=False):\n",
    "    dump_file = './feats/user_time_delta_feature.pkl'\n",
    "\n",
    "    if os.path.exists(dump_file) and not updata:\n",
    "        print('Found ' + dump_file)\n",
    "    else:\n",
    "        print('gen_user_time_delta_feature ing...')\n",
    "        dfal = get_nominal_dfal()\n",
    "        dfal = dfal[['user_id', 'dt']].sort_values('dt', ascending=True)\n",
    "        ugp = dfal.groupby('user_id')\n",
    "        \n",
    "        threshold_3day = 60 * 60 * 24 * 3\n",
    "        threshold_2day = 60 * 60 * 24 * 2\n",
    "        threshold_1day = 60 * 60 * 24 * 1\n",
    "        \n",
    "        dfal['agg_user_ts_delta1'] = ugp['dt'].transform(lambda x: x.diff(1))\n",
    "        dfal['agg_user_new'] = dfal.agg_user_ts_delta1.isnull().apply(lambda x: 1 if x else 0)\n",
    "        dfal.agg_user_ts_delta1 = dfal.agg_user_ts_delta1.apply(lambda x: x.total_seconds())\n",
    "        dfal.agg_user_ts_delta1 = dfal.agg_user_ts_delta1.apply(lambda x: x if x < threshold_3day else threshold)\n",
    "        dfal.agg_user_ts_delta1 = dfal.agg_user_ts_delta1.fillna(threshold_3day)\n",
    "        dfal['agg_user_da_delta1'] = dfal.agg_user_ts_delta1.apply(lambda x:x//86400)\n",
    "        dfal['agg_user_ho_delta1'] = dfal.agg_user_ts_delta1.apply(lambda x:x//3600)\n",
    "        dfal['agg_user_mi_delta1'] = dfal.agg_user_ts_delta1.apply(lambda x:x//60)\n",
    "        \n",
    "        #dfal['agg_user_noa_05mi'] = dfal.agg_user_mi_delta.apply(lambda x: 1 if x >=  5 else 0)\n",
    "        #dfal['agg_user_noa_10mi'] = dfal.agg_user_mi_delta.apply(lambda x: 1 if x >= 10 else 0)\n",
    "        #dfal['agg_user_noa_30mi'] = dfal.agg_user_mi_delta.apply(lambda x: 1 if x >= 30 else 0)\n",
    "        \n",
    "        #dfal['agg_user_noa_2ho'] = dfal.agg_user_ho_delta.apply(lambda x: 1 if x >= 2 else 0)\n",
    "        #dfal['agg_user_noa_5ho'] = dfal.agg_user_ho_delta.apply(lambda x: 1 if x >= 5 else 0)\n",
    "        #dfal['agg_user_noa_9ho'] = dfal.agg_user_ho_delta.apply(lambda x: 1 if x >= 9 else 0)\n",
    "        \n",
    "        #dfal['agg_user_noa_1da'] = dfal.agg_user_ts_delta.apply(lambda x: 1 if x >= threshold_1day else 0)\n",
    "        #dfal['agg_user_noa_2da'] = dfal.agg_user_ts_delta.apply(lambda x: 1 if x >= threshold_2day else 0)\n",
    "        #dfal['agg_user_noa_3da'] = dfal.agg_user_ts_delta.apply(lambda x: 1 if x >= threshold_3day else 0)\n",
    "        dfal = dfal.drop(['dt', 'user_id'], axis=1)\n",
    "        dfal, _ = reduce_mem_usage(dfal)\n",
    "        dump_pickle(dfal, dump_file)\n",
    "        print('gen_user_time_delta_feature completed')\n",
    "        del dfal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-22T02:08:23.477607Z",
     "start_time": "2018-04-22T02:08:23.406400Z"
    }
   },
   "outputs": [],
   "source": [
    "def gen_user_last_attrs_feature(updata=False):\n",
    "    dump_file = './feats/user_last_attrs_feature.pkl'\n",
    "    if os.path.exists(dump_file) and not updata:\n",
    "        print('Found ' + dump_file)\n",
    "    else:\n",
    "        print('gen_user_last_attrs_feature ing...')\n",
    "        attrs_cols = [\n",
    "            'item_category_list', 'item_brand_id', 'item_city_id', 'item_price_level',\n",
    "            'item_sales_level', 'item_collected_level', 'item_pv_level',\n",
    "            'context_page_id', 'shop_review_num_level', 'shop_star_level',\n",
    "            'shop_review_positive_rate', 'shop_score_delivery',\n",
    "            'shop_score_description', 'shop_score_service'\n",
    "        ]\n",
    "\n",
    "        level_cols = [\n",
    "            'item_price_level', 'item_sales_level', 'item_collected_level',\n",
    "            'item_pv_level', 'shop_review_num_level', 'shop_star_level',\n",
    "            'shop_review_positive_rate', 'shop_score_delivery',\n",
    "            'shop_score_description', 'shop_score_service'\n",
    "        ]\n",
    "        dfal = get_nominal_dfal()\n",
    "        dfal = dfal[['user_id', 'dt'] + attrs_cols].sort_values('dt', ascending=True)\n",
    "        ugp = dfal.groupby('user_id')\n",
    "\n",
    "        dfal[['agg_last1_' + c for c in attrs_cols]] = ugp[attrs_cols].transform(lambda x: x.shift(1))\n",
    "        dfal[['agg_last2_' + c for c in attrs_cols]] = ugp[attrs_cols].transform(lambda x: x.shift(2))\n",
    "        dfal[['agg_diff1_' + c for c in level_cols]] = ugp[level_cols].transform(lambda x: x.diff(1))\n",
    "        dfal[['agg_diff2_' + c for c in level_cols]] = ugp[level_cols].transform(lambda x: x.diff(2))\n",
    "        dfal = dfal.drop(attrs_cols + ['dt', 'user_id'], axis=1)\n",
    "        dump_pickle(dfal, dump_file)\n",
    "        print('gen_user_last_attrs_feature completed')\n",
    "        del dfal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-04-22T02:12:14.437412Z",
     "start_time": "2018-04-22T02:08:23.532144Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gen_user_time_delta_feature ing...\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'threshold' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-ff9a72b49613>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0m__name__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'__main__'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mgen_user_time_delta_feature\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0mgen_user_last_attrs_feature\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-eee35d7ed923>\u001b[0m in \u001b[0;36mgen_user_time_delta_feature\u001b[0;34m(updata)\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mdfal\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'agg_user_new'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnull\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_seconds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mthreshold_3day\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mthreshold\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m         \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthreshold_3day\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mdfal\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'agg_user_da_delta1'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m//\u001b[0m\u001b[0;36m86400\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.pyenv/versions/stats36/lib/python3.6/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[1;32m   2353\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2354\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masobject\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2355\u001b[0;31m                 \u001b[0mmapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2356\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2357\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/src/inference.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer (pandas/_libs/lib.c:66645)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-eee35d7ed923>\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mdfal\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'agg_user_new'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnull\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtotal_seconds\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mthreshold_3day\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mthreshold\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m         \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfillna\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mthreshold_3day\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mdfal\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'agg_user_da_delta1'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdfal\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0magg_user_ts_delta1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m//\u001b[0m\u001b[0;36m86400\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'threshold' is not defined"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    gen_user_time_delta_feature(True)\n",
    "    gen_user_last_attrs_feature(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "stats36",
   "language": "python",
   "name": "stats36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}