xgboost.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9da94b1c",
   "metadata": {},
   "source": [
    "## XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "de2ec5d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import seaborn as sns\n",
    "import scipy as sp\n",
    "import datetime as dt\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn import linear_model\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.preprocessing import scale\n",
    "from collections import Counter\n",
    "import ziptotimezone as zip_helper\n",
    "from sklearn.model_selection import train_test_split\n",
    "import xgboost as xgb\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import graphviz \n",
    "import matplotlib.pyplot as plt\n",
    "import mpu\n",
    "from uszipcode import SearchEngine\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1dcabf5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv('eBay_ML_Challenge_Dataset_2021_train.csv', nrows=500000)\n",
    "#dataset = pd.read_csv(\"partially_processed_data.csv\", nrows=500000)\n",
    "#other_dataset = pd.read_csv('eBay_ML_Challenge_Dataset_2021_train.csv', nrows=500000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9bf1c2df",
   "metadata": {},
   "outputs": [],
   "source": [
    "b2c_c2c = np.array(dataset[\"b2c_c2c\"])\n",
    "if b2c_c2c[0] in [0,1]:\n",
    "        print(\"Array has already been converted to numeric binary!\")\n",
    "else:\n",
    "    for i in range(len(b2c_c2c)):\n",
    "        if b2c_c2c[i][0] == \"B\":\n",
    "            b2c_c2c[i] = 0\n",
    "        else:\n",
    "            b2c_c2c[i] = 1\n",
    "dataset[\"b2c_c2c\"] = b2c_c2c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "55506df0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def round_datetime_to_date(datetime):\n",
    "    days = datetime.days\n",
    "    hours = datetime.seconds // 3600\n",
    "    if hours > 12:\n",
    "        return days + 1\n",
    "    else:\n",
    "        return days\n",
    "\n",
    "def calculate_handling_and_delivery_days(acceptance_timestamps, payment_timestamps, delivery_date):\n",
    "    handling_labels = []\n",
    "    shipping_labels = []\n",
    "    delivery_labels = []\n",
    "    for i in range(acceptance_timestamps.shape[0]):\n",
    "        raw_payment = payment_timestamps[i]\n",
    "        raw_acceptance = acceptance_timestamps[i]\n",
    "        #parse raw_payment time string to separate year, month, date, and time\n",
    "        p_year, p_month, p_date = int(raw_payment[0:4]), int(raw_payment[5:7]), int(raw_payment[8:10])\n",
    "        p_hour, p_min, p_sec = int(raw_payment[11:13]), int(raw_payment[14:16]), int(raw_payment[17:19])\n",
    "        p_datetime = dt.datetime(year=p_year, month=p_month, day=p_date, hour=p_hour, minute=p_min, second=p_sec)\n",
    "            \n",
    "        #parse raw_acceptance time string to separate year, month, date, and time\n",
    "        raw_acceptance = acceptance_timestamps[i]\n",
    "        a_year, a_month, a_date = int(raw_acceptance[0:4]), int(raw_acceptance[5:7]), int(raw_acceptance[8:10])\n",
    "        a_hour, a_min, a_sec = int(raw_acceptance[11:13]), int(raw_acceptance[14:16]), int(raw_acceptance[17:19])\n",
    "        a_datetime = dt.datetime(year=a_year, month=a_month, day=a_date, hour=a_hour, minute=a_min, second=a_sec)\n",
    "        \n",
    "        raw_delivery = delivery_date[i]\n",
    "        d_year, d_month, d_date = int(raw_delivery[0:4]), int(raw_delivery[5:7]), int(raw_delivery[8:10])\n",
    "        d_date = dt.datetime(year=d_year, month=d_month, day=d_date, hour=17)\n",
    "        \n",
    "        #handling days = acceptance time - payment time; shipping days = delivery date - acceptance time\n",
    "        handling_days = a_datetime - p_datetime\n",
    "        shipping_days = d_date - a_datetime\n",
    "        delivery_days = d_date - p_datetime\n",
    "        \n",
    "        #round to nearest day\n",
    "        rounded_handling_days = round_datetime_to_date(handling_days)\n",
    "        rounded_shipping_days = round_datetime_to_date(shipping_days)\n",
    "        rounded_delivery_days = round_datetime_to_date(delivery_days)\n",
    "        \n",
    "        handling_labels.append(rounded_handling_days)\n",
    "        shipping_labels.append(rounded_shipping_days)\n",
    "        delivery_labels.append(rounded_delivery_days)\n",
    "        \n",
    "    return np.array(handling_labels), np.array(shipping_labels), np.array(delivery_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70958c81",
   "metadata": {},
   "outputs": [],
   "source": [
    "handling_days, shipping_days, delivery_days = calculate_handling_and_delivery_days(dataset[\"acceptance_scan_timestamp\"], dataset[\"payment_datetime\"], dataset[\"delivery_date\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4afa6d5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "weight = dataset[\"weight\"]\n",
    "weight_units = dataset[\"weight_units\"]\n",
    "for i, unit in enumerate(weight_units):\n",
    "    if unit == 2:\n",
    "        #convert weight to lbs; 1 kg = 2.20462 lbs.\n",
    "        weight[i] *= 2.20462\n",
    "dataset[\"weight\"] = weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24bfbdad",
   "metadata": {},
   "outputs": [],
   "source": [
    "weight = dataset[\"weight\"]\n",
    "category_id = dataset[\"category_id\"]\n",
    "\n",
    "#determine average weight by category ID \n",
    "category_id_weights = {}\n",
    "for i, w in enumerate(weight):\n",
    "    category = category_id[i]\n",
    "    if category not in category_id_weights:\n",
    "        category_id_weights[category] = [w]\n",
    "    else:\n",
    "        category_id_weights[category].append(w)\n",
    "\n",
    "category_id_weight_means = {}\n",
    "for category in category_id_weights:\n",
    "    weights = category_id_weights[category]\n",
    "    average_weight = np.mean(weights)\n",
    "    category_id_weight_means[category] = average_weight\n",
    "\n",
    "#fill missing weights\n",
    "weight_means = category_id_weight_means\n",
    "overall_mean = np.mean(weight)\n",
    "for i, w in enumerate(weight):\n",
    "    if w == 0:\n",
    "        #weight is missing, replace with average weight across same category id\n",
    "        category = category_id[i]\n",
    "        if category in weight_means:\n",
    "            weight[i] = weight_means[category]\n",
    "        else:\n",
    "            #don't have records for this category id, so replace with overall average\n",
    "            weight[i] = overall_mean\n",
    "\n",
    "dataset[\"weight\"] = weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f89db41",
   "metadata": {},
   "outputs": [],
   "source": [
    "#package_size = dataset[\"package_size\"]\n",
    "\n",
    "#if type(package_size[0]) == int:\n",
    "        #print(\"Already converted to discrete numeric values\")\n",
    "#else:\n",
    "#encodings = {\"LETTER\": 0, \"PACKAGE_THICK_ENVELOPE\": 1, \"LARGE_ENVELOPE\": 2,\"VERY_LARGE_PACKAGE\": 3, \n",
    "             #\"LARGE_PACKAGE\": 4, \"EXTRA_LARGE_PACKAGE\": 5, \"NONE\": -1}\n",
    "#for i, size in enumerate(package_size):\n",
    "    #if type(package_size[i]) != int:\n",
    "        #package_size[i] = encodings[size]\n",
    "\n",
    "#dataset[\"package_size\"] = package_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17f81d6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.drop([\"package_size\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9bf426b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# package_size = dataset[\"package_size\"]\n",
    "# weight = dataset[\"weight\"]\n",
    "\n",
    "#determine average weight by package size\n",
    "# package_size_weights = {}\n",
    "# for i, w in enumerate(weight):\n",
    "#     p_size = package_size[i]\n",
    "#     if p_size not in package_size_weights:\n",
    "#         package_size_weights[p_size] = [w]\n",
    "#     else:\n",
    "#         package_size_weights[p_size].append(w)\n",
    "\n",
    "# package_id_weight_means = {}\n",
    "# for p_size in package_size_weights:\n",
    "#     weights = package_size_weights[p_size]\n",
    "#     average_weight = np.mean(weights)\n",
    "#     package_id_weight_means[p_size] = average_weight\n",
    "\n",
    "# #fill in missing package sizes\n",
    "# weight_means = package_id_weight_means\n",
    "# weight_means.pop(-1, None)\n",
    "# weight_means_list = [weight_means[key] for key in weight_means]\n",
    "# for i, s in enumerate(package_size):\n",
    "#     if s == -1:\n",
    "#         #package size is missing, replace with package size it's weight is closest to the average of\n",
    "#         w = weight[i]\n",
    "#         abs_function = lambda value: abs(value-w)\n",
    "#         closest_value = min(weight_means_list, key=abs_function)\n",
    "#         closest_p_size = weight_means_list.index(closest_value)\n",
    "#         package_size[i] = closest_p_size\n",
    "\n",
    "# dataset[\"package_size\"] = package_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f881aa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "shipment_method_id = dataset[\"shipment_method_id\"]\n",
    "carrier_min_estimate = dataset[\"carrier_min_estimate\"]\n",
    "carrier_max_estimate = dataset[\"carrier_max_estimate\"]\n",
    "\n",
    "#determine average shipping estimates by shipping method   \n",
    "carrier_min_by_shipment_method = {}\n",
    "carrier_max_by_shipment_method = {}\n",
    "for i, method_id in enumerate(shipment_method_id):\n",
    "    carrier_min = carrier_min_estimate[i]\n",
    "    carrier_max = carrier_max_estimate[i]\n",
    "    if method_id not in carrier_min_by_shipment_method:\n",
    "        carrier_min_by_shipment_method[method_id] = [carrier_min]\n",
    "    else:\n",
    "        carrier_min_by_shipment_method[method_id].append(carrier_min)\n",
    "\n",
    "    if method_id not in carrier_max_by_shipment_method:\n",
    "        carrier_max_by_shipment_method[method_id] = [carrier_max]\n",
    "    else:\n",
    "        carrier_max_by_shipment_method[method_id].append(carrier_max)\n",
    "\n",
    "carrier_min_means = {}\n",
    "for method_id in carrier_min_by_shipment_method:\n",
    "    min_estimates = carrier_min_by_shipment_method[method_id]\n",
    "    mean_min_estimate = np.mean(min_estimates)\n",
    "    carrier_min_means[method_id] = mean_min_estimate\n",
    "\n",
    "carrier_max_means = {}\n",
    "for method_id in carrier_max_by_shipment_method:\n",
    "    max_estimates = carrier_max_by_shipment_method[method_id]\n",
    "    mean_max_estimate = np.mean(max_estimates)\n",
    "    carrier_max_means[method_id] = mean_max_estimate \n",
    "    \n",
    "#fill in missing estimates\n",
    "overall_min_mean, overall_max_mean = np.mean(carrier_min_estimate), np.mean(carrier_max_estimate)\n",
    "for i, estimate in enumerate(carrier_min_estimate):\n",
    "    if estimate < 0:\n",
    "        #need to fill value \n",
    "        method_id = shipment_method_id[i]\n",
    "        if method_id in carrier_min_means:\n",
    "            carrier_min_estimate[i] = carrier_min_means[method_id]\n",
    "        else:\n",
    "            carrier_min_estimate[i] = overall_min_mean\n",
    "for i, estimate in enumerate(carrier_max_estimate):\n",
    "    if estimate < 0:\n",
    "        #need to fill value\n",
    "        method_id = shipment_method_id[i]\n",
    "        if method_id in carrier_max_means:\n",
    "            carrier_max_estimate[i] = carrier_max_means[method_id]\n",
    "        else:\n",
    "            carrier_max_estimate[i] = overall_max_mean\n",
    "\n",
    "dataset[\"carrier_min_estimate\"] = carrier_min_estimate\n",
    "dataset[\"carrier_max_estimate\"] = carrier_max_estimate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f7110b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#SINCE DECLARED HANDLING DAYS IS THE MOST IMPORTANT FEATURE, I DON'T THINK \n",
    "#THIS IS A GOOD DATA FEATURE TO FILL WITH A NAIVE AVERAGE. \n",
    "#declared_handling_days = dataset[\"declared_handling_days\"]\n",
    "#seller_id = dataset[\"seller_id\"]\n",
    "#def fill_missing_declared_handling_days():\n",
    "#overall_mean = np.mean(declared_handling_days)\n",
    "#seller_counts = Counter(seller_id)\n",
    "#for i, days in enumerate(declared_handling_days):\n",
    "    #if np.isnan(days):\n",
    "        #need to fill\n",
    "        #declared_handling_days[i] = overall_mean\n",
    "\n",
    "#dataset[\"declared_handling_days\"] = declared_handling_days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b785d0a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[\"handling_days\"] = handling_days\n",
    "dataset.drop([\"acceptance_scan_timestamp\"], axis=1, inplace=True)\n",
    "dataset.drop([\"payment_datetime\"], axis=1, inplace=True)\n",
    "dataset.drop([\"delivery_date\"], axis=1, inplace=True)\n",
    "dataset.drop([\"weight_units\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bdd5cf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_zip_features(item_zip, buyer_zip):\n",
    "    \"\"\"\n",
    "    Haversine formula using 'mpu' library which determines the\n",
    "    great-circle distance between two points on a sphere.\n",
    "    \"\"\"\n",
    "    if item_zip is not None and buyer_zip is not None:\n",
    "        search = SearchEngine(simple_zipcode=True)\n",
    "\n",
    "        zip1 = search.by_zipcode(item_zip[0:5])\n",
    "        lat1 =zip1.lat\n",
    "        long1 =zip1.lng\n",
    "        pop_density1 = zip1.population_density\n",
    "        median_income1 = zip1.median_household_income\n",
    "\n",
    "        zip2 =search.by_zipcode(buyer_zip[0:5])\n",
    "        lat2 =zip2.lat\n",
    "        long2 =zip2.lng\n",
    "        pop_density2 = zip2.population_density\n",
    "        median_income2 = zip2.median_household_income\n",
    "\n",
    "        if lat1 is None or lat2 is None or long1 is None or long2 is None:\n",
    "            lat1, long1 = zip_helper.zip_to_central_lat_lon(int(item_zip))\n",
    "            lat2, long2 = zip_helper.zip_to_central_lat_lon(int(buyer_zip))\n",
    "                \n",
    "        return mpu.haversine_distance((lat1,long1),(lat2,long2)), pop_density1, median_income1, pop_density2, median_income2\n",
    "    else:\n",
    "        print(\"item zip or buyer zip was None\")\n",
    "        return None\n",
    "\n",
    "\n",
    "def add_zip_feature_columns(item_zip, buyer_zip, dataset):\n",
    "    #item_zip_str = item_zip.apply(lambda x: str(x))\n",
    "    #buyer_zip_str = buyer_zip.apply(lambda x: str(x))\n",
    "\n",
    "    #zips = pd.concat([item_zip_str, buyer_zip_str], axis=1)\n",
    "    zips = pd.concat([item_zip, buyer_zip], axis=1)\n",
    "    item_z = zips[\"item_zip\"]\n",
    "    zips[\"distance\"] = [0] * zips.shape[0]\n",
    "    zips[\"item_zip_pop_density\"] = [0] * zips.shape[0]\n",
    "    zips[\"item_zip_median_income\"] = [0] * zips.shape[0]\n",
    "    zips[\"buyer_zip_pop_density\"] = [0] * zips.shape[0]\n",
    "    zips[\"buyer_zip_median_income\"] = [0] * zips.shape[0]\n",
    "    remove_indeces = []\n",
    "    for i, z in enumerate(item_z):\n",
    "        if i%1000 == 0:\n",
    "            print(\"on data instance number \" + str(i))\n",
    "        try:\n",
    "            distance, pop_density1, median_income1, pop_density2, median_income2 = get_zip_features(z, zips[\"buyer_zip\"][i])\n",
    "            zips[\"distance\"][i] = distance\n",
    "            zips[\"item_zip_pop_density\"][i] = pop_density1\n",
    "            zips[\"item_zip_median_income\"][i] = median_income1\n",
    "            zips[\"buyer_zip_pop_density\"][i] = pop_density2\n",
    "            zips[\"buyer_zip_median_income\"][i] = median_income2\n",
    "        except Exception as e:\n",
    "            remove_indeces.append(i)\n",
    "    dataset.drop(remove_indeces, inplace=True, axis=0)\n",
    "    labels = pd.DataFrame(delivery_days, columns=[\"delivery_days\"])\n",
    "    labels.drop(remove_indeces, inplace=True, axis=0)\n",
    "    zips.drop(remove_indeces, inplace=True, axis=0)\n",
    "            \n",
    "    #zips['distance'] = zips.apply(lambda x: get_distance(x.item_zip, x.buyer_zip), axis=1)\n",
    "    return zips['distance'], zips[\"item_zip_pop_density\"], zips[\"item_zip_median_income\"], zips[\"buyer_zip_pop_density\"], zips[\"buyer_zip_median_income\"], labels\n",
    "\n",
    "distance, item_density, item_income, buyer_density, buyer_income, labels = add_zip_feature_columns(dataset[\"item_zip\"], dataset[\"buyer_zip\"], dataset)\n",
    "dataset[\"zip_distance\"] = distance\n",
    "dataset[\"item_zip_pop_density\"] = item_density\n",
    "dataset[\"item_zip_median_income\"] = item_income\n",
    "dataset[\"buyer_zip_pop_density\"] = buyer_density\n",
    "dataset[\"buyer_zip_median_income\"] = buyer_income\n",
    "dataset[\"labels\"] = labels"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ce028f8",
   "metadata": {},
   "source": [
    "ON DATA INSTANCE NUMBER 1174000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2596c8ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.to_csv(\"partially_processed_data.csv\", sep=\",\", header=dataset.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24bbf5a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57f829a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1000000):\n",
    "    try: \n",
    "        if type(dataset[\"item_zip\"][i]) != int:\n",
    "            dataset[\"item_zip\"][i] = int(dataset[\"item_zip\"][i][0:5])\n",
    "            dataset[\"buyer_zip\"][i] = int(dataset[\"buyer_zip\"][i][0:5])\n",
    "    except Exception as e:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f559b9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = dataset.to_numpy()\n",
    "labels = np.array(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41df1b67",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bc0c310",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, Y_train, Y_test = train_test_split(features, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e996eb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = scale(X_train, with_mean=True, with_std=True)\n",
    "X_test = scale(X_test, with_mean=True, with_std=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "caabed65",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_loss(preds, actual):\n",
    "    early_loss, late_loss = 0,0 \n",
    "    for i in range(len(preds)):\n",
    "        if preds[i] < actual[i]:\n",
    "            #early shipment\n",
    "            early_loss += actual[i] - preds[i]\n",
    "        elif preds[i] > actual[i]:\n",
    "            #late shipment\n",
    "            late_loss += preds[i] - actual[i]\n",
    "    loss = (1/len(preds)) * (0.4 * (early_loss) + 0.6 * (late_loss))\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a1452d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Before fine tuning\")\n",
    "xgbr = xgb.XGBRegressor(verbosity=0)\n",
    "print(xgbr)\n",
    "xgbr.fit(X_train, Y_train)\n",
    "train_score = xgbr.score(X_train, Y_train)\n",
    "print(\"train score: \" + str(train_score))\n",
    "pred = xgbr.predict(X_test)\n",
    "loss = evaluate_loss(pred, Y_test)\n",
    "print(\"loss: \" + str(loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d462656a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "print(\"Tuning\")\n",
    "\n",
    "params = {\n",
    "    # Parameters that we are going to tune.\n",
    "    'max_depth':[int(x) for x in np.linspace(start=1, stop=20, num=1)],\n",
    "    'min_child_weight':[0,1,2,3,4,5,6,7,8,9,10],\n",
    "    'eta':[0.5, 0.4, 0.3, 0.2, 0.1, 0.05, 0.01, 0.005],\n",
    "    'subsample': [x/100 for x in np.linspace(start=1, stop=100, num=2)],\n",
    "    'colsample_bytree': [x/100 for x in np.linspace(start=1, stop=100, num=2)],\n",
    "}\n",
    "\n",
    "xgb_random_search = RandomizedSearchCV(estimator=xgbr,\n",
    "                                      param_distributions = params,\n",
    "                                      n_iter = 100,\n",
    "                                      cv=3,\n",
    "                                      verbose=2,\n",
    "                                      random_state=47,\n",
    "                                      n_jobs=2)\n",
    "\n",
    "xgb_random_search.fit(X_train, Y_train)\n",
    "print(xgb_random_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a2f60da",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params = xgb_random_search.best_params_\n",
    "subsample = best_params[\"subsample\"]\n",
    "min_child_weight = best_params[\"min_child_weight\"]\n",
    "max_depth = best_params[\"max_depth\"]\n",
    "eta = best_params[\"eta\"]\n",
    "colsample_by_tree = best_params[\"colsample_bytree\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72ebf09f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"After fine tuning\")\n",
    "xgbr = xgb.XGBRegressor(verbosity=0, subsample=subsample, min_child_weight=min_child_weight,\n",
    "                       max_depth = max_depth, eta=eta, colsample_by_tree=colsample_by_tree)\n",
    "print(xgbr)\n",
    "xgbr.fit(X_train, Y_train)\n",
    "train_score = xgbr.score(X_train, Y_train)\n",
    "print(\"train score: \" + str(train_score))\n",
    "pred = xgbr.predict(X_test)\n",
    "loss = evaluate_loss(pred, Y_test)\n",
    "print(\"loss: \" + str(loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ab7afdb",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(dataset.columns.shape[0]):\n",
    "    print(dataset.columns[i] + \": \" + str(xgbr.feature_importances_[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90e38e25",
   "metadata": {},
   "source": [
    "Let's see if we can improve performance by dropping features who have an importance of <0.3\n",
    "\n",
    "b2c_c2c: 0.015656047\n",
    "seller_id: 0.047507692\n",
    "declared_handling_days: 0.042284053\n",
    "shipment_method_id: 0.035130586\n",
    "shipping_fee: 0.028294528\n",
    "carrier_min_estimate: 0.044301696\n",
    "carrier_max_estimate: 0.055198833\n",
    "item_zip: 0.0597363\n",
    "buyer_zip: 0.028275775\n",
    "category_id: 0.022790782\n",
    "item_price: 0.043802027\n",
    "quantity: 0.04765062\n",
    "weight: 0.05730528\n",
    "package_size: 0.020258492\n",
    "record_number: 0.022046657\n",
    "handling_days: 0.2548077\n",
    "zip_distance: 0.04024559\n",
    "item_zip_pop_density: 0.026283627\n",
    "item_zip_median_income: 0.04863616\n",
    "buyer_zip_pop_density: 0.03418408\n",
    "buyer_zip_median_income: 0.025603488"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8cb2d9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.drop([\"b2c_c2c\", \"quantity\", \"shipping_fee\", \"category_id\", \"package_size\", \"record_number\", \"item_zip_pop_density\", \"buyer_zip_median_income\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed6fb0eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "48b0db15",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = dataset.to_numpy()\n",
    "labels = np.array(labels)\n",
    "X_train = scale(X_train, with_mean=True, with_std=True)\n",
    "X_test = scale(X_test, with_mean=True, with_std=True)\n",
    "\n",
    "print(\"Before fine tuning\")\n",
    "xgbr = xgb.XGBRegressor(verbosity=0)\n",
    "print(xgbr)\n",
    "xgbr.fit(X_train, Y_train)\n",
    "train_score = xgbr.score(X_train, Y_train)\n",
    "print(\"train score: \" + str(train_score))\n",
    "pred = xgbr.predict(X_test)\n",
    "loss = evaluate_loss(pred, Y_test)\n",
    "print(\"loss: \" + str(loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "183bc701",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "print(\"Tuning\")\n",
    "\n",
    "params = {\n",
    "    # Parameters that we are going to tune.\n",
    "    'max_depth':[int(x) for x in np.linspace(start=5, stop=20, num=1)],\n",
    "    'min_child_weight':[int(x) for x in np.linspace(start=1, stop=10, num=1)],\n",
    "    'eta':[0.3, 0.2, 0.1, 0.05, 0.01, 0.005],\n",
    "    'subsample': [x/10 for x in np.linspace(start=1, stop=10, num=1)],\n",
    "    'colsample_bytree': [x/10 for x in np.linspace(start=1, stop=10, num=1)],\n",
    "    'n_estimators': [int(x) for x in np.linspace(start=50, stop=500, num=50)]\n",
    "}\n",
    "\n",
    "xgb_random_search = RandomizedSearchCV(estimator=xgbr,\n",
    "                                      param_distributions = params,\n",
    "                                      n_iter = 100,\n",
    "                                      cv=3,\n",
    "                                      verbose=2,\n",
    "                                      random_state=47,\n",
    "                                      n_jobs=2)\n",
    "\n",
    "xgb_random_search.fit(X_train, Y_train)\n",
    "print(xgb_random_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "188d986a",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params = xgb_random_search.best_params_\n",
    "subsample = best_params[\"subsample\"]\n",
    "min_child_weight = best_params[\"min_child_weight\"]\n",
    "max_depth = best_params[\"max_depth\"]\n",
    "eta = best_params[\"eta\"]\n",
    "colsample_by_tree = best_params[\"colsample_bytree\"]\n",
    "n_estimators = best_params[\"n_estimators\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0841fc6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"After fine tuning\")\n",
    "xgbr = xgb.XGBRegressor(verbosity=0, subsample=subsample, min_child_weight=min_child_weight,\n",
    "                       max_depth = max_depth, eta=eta, colsample_by_tree=colsample_by_tree)\n",
    "print(xgbr)\n",
    "xgbr.fit(X_train, Y_train)\n",
    "train_score = xgbr.score(X_train, Y_train)\n",
    "print(\"train score: \" + str(train_score))\n",
    "pred = xgbr.predict(X_test)\n",
    "loss = evaluate_loss(pred, Y_test)\n",
    "print(\"loss: \" + str(loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "258f854d",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(dataset.columns.shape[0]):\n",
    "    print(dataset.columns[i] + \": \" + str(xgbr.feature_importances_[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9809becd",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "664e6a4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.drop([\"declared_handling_days\", \"shipment_method_id\", \"carrier_min_estimate\",\n",
    "             \"item_zip\", \"buyer_zip\", \"weight\"], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e661923d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4deb06d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = dataset.to_numpy()\n",
    "labels = np.array(labels)\n",
    "X_train = scale(X_train, with_mean=True, with_std=True)\n",
    "X_test = scale(X_test, with_mean=True, with_std=True)\n",
    "\n",
    "print(\"Before fine tuning\")\n",
    "xgbr = xgb.XGBRegressor(verbosity=0)\n",
    "print(xgbr)\n",
    "xgbr.fit(X_train, Y_train)\n",
    "train_score = xgbr.score(X_train, Y_train)\n",
    "print(\"train score: \" + str(train_score))\n",
    "pred = xgbr.predict(X_test)\n",
    "loss = evaluate_loss(pred, Y_test)\n",
    "print(\"loss: \" + str(loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94a52060",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "print(\"Tuning\")\n",
    "\n",
    "params = {\n",
    "    # Parameters that we are going to tune.\n",
    "    'max_depth':[int(x) for x in np.linspace(start=5, stop=20, num=1)],\n",
    "    'min_child_weight':[int(x) for x in np.linspace(start=1, stop=10, num=1)],\n",
    "    'eta':[0.3, 0.2, 0.1, 0.05, 0.01, 0.005],\n",
    "    'subsample': [x/10 for x in np.linspace(start=1, stop=10, num=1)],\n",
    "    'colsample_bytree': [x/10 for x in np.linspace(start=1, stop=10, num=1)],\n",
    "    'n_estimators': [int(x) for x in np.linspace(start=50, stop=500, num=50)]\n",
    "}\n",
    "\n",
    "xgb_random_search = RandomizedSearchCV(estimator=xgbr,\n",
    "                                      param_distributions = params,\n",
    "                                      n_iter = 100,\n",
    "                                      cv=3,\n",
    "                                      verbose=2,\n",
    "                                      random_state=47,\n",
    "                                      n_jobs=2)\n",
    "\n",
    "xgb_random_search.fit(X_train, Y_train)\n",
    "print(xgb_random_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c43b893",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_params = xgb_random_search.best_params_\n",
    "subsample = best_params[\"subsample\"]\n",
    "min_child_weight = best_params[\"min_child_weight\"]\n",
    "max_depth = best_params[\"max_depth\"]\n",
    "eta = best_params[\"eta\"]\n",
    "colsample_by_tree = best_params[\"colsample_bytree\"]\n",
    "n_estimators = best_params[\"n_estimators\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18df7ddf",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"After fine tuning\")\n",
    "tuned_xgbr = xgb.XGBRegressor(verbosity=0, subsample=subsample, min_child_weight=min_child_weight,\n",
    "                       max_depth = max_depth, eta=eta, colsample_by_tree=colsample_by_tree,\n",
    "                       n_estimators = n_estimators)\n",
    "print(xgbr)\n",
    "tuned_xgbr.fit(X_train, Y_train)\n",
    "train_score = tuned_xgbr.score(X_train, Y_train)\n",
    "print(\"train score: \" + str(train_score))\n",
    "pred = tuned_xgbr.predict(X_test)\n",
    "loss = evaluate_loss(pred, Y_test)\n",
    "print(\"loss: \" + str(loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "71d396ac",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc4e3c58",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}