Multiclass_Classification_and_Regression with different techniques to avoid overfitting.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Multiclass Classification and Regression.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNRg0VWXp3xITi31pnrCTel",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/nasiksami/Deep-Learning/blob/main/Multiclass_Classification_and_Regression%20with%20different%20techniques%20to%20avoid%20overfitting.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1bup0GQsYxlK"
      },
      "source": [
        "Classification problem using zscore \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VNxa0HoT1B5l"
      },
      "source": [
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['age'] = zscore(df['age'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('product').drop('id')\n",
        "x = df[x_columns].values\n",
        "dummies = pd.get_dummies(df['product']) # Classification\n",
        "products = dummies.columns\n",
        "y = dummies.values"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "40GdFN7aA7UF",
        "outputId": "d651527b-8c59-438f-c430-9512e9079136",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "\n",
        "# Classification neural network\n",
        "import numpy as np\n",
        "import tensorflow.keras\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Split into train/test\n",
        "x_train, x_test, y_train, y_test = train_test_split(    \n",
        "    x, y, test_size=0.25, random_state=42)\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Dense(150, input_dim=x.shape[1], activation='relu',\n",
        "                kernel_initializer='random_normal'))\n",
        "model.add(Dense(100,activation='relu',kernel_initializer='random_normal'))\n",
        "model.add(Dense(50,activation='relu',kernel_initializer='random_normal'))\n",
        "model.add(Dense(y.shape[1],activation='softmax',\n",
        "                kernel_initializer='random_normal'))\n",
        "model.compile(loss='categorical_crossentropy', \n",
        "              optimizer=tensorflow.keras.optimizers.Adam(),\n",
        "              metrics =['accuracy'])\n",
        "monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=5, \n",
        "                        verbose=1, mode='auto', restore_best_weights=True)\n",
        "model.fit(x_train,y_train,validation_data=(x_test,y_test),\n",
        "          callbacks=[monitor],verbose=2,epochs=100)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/100\n",
            "47/47 - 0s - loss: 1.4006 - accuracy: 0.4273 - val_loss: 1.1240 - val_accuracy: 0.4980\n",
            "Epoch 2/100\n",
            "47/47 - 0s - loss: 1.0948 - accuracy: 0.5160 - val_loss: 1.0161 - val_accuracy: 0.5640\n",
            "Epoch 3/100\n",
            "47/47 - 0s - loss: 0.9165 - accuracy: 0.6273 - val_loss: 0.8690 - val_accuracy: 0.6700\n",
            "Epoch 4/100\n",
            "47/47 - 0s - loss: 0.8130 - accuracy: 0.6747 - val_loss: 0.8232 - val_accuracy: 0.6660\n",
            "Epoch 5/100\n",
            "47/47 - 0s - loss: 0.7710 - accuracy: 0.6813 - val_loss: 0.7991 - val_accuracy: 0.6860\n",
            "Epoch 6/100\n",
            "47/47 - 0s - loss: 0.7558 - accuracy: 0.6940 - val_loss: 0.8458 - val_accuracy: 0.6360\n",
            "Epoch 7/100\n",
            "47/47 - 0s - loss: 0.7381 - accuracy: 0.6820 - val_loss: 0.8098 - val_accuracy: 0.6680\n",
            "Epoch 8/100\n",
            "47/47 - 0s - loss: 0.7289 - accuracy: 0.6940 - val_loss: 0.7573 - val_accuracy: 0.6880\n",
            "Epoch 9/100\n",
            "47/47 - 0s - loss: 0.7039 - accuracy: 0.7007 - val_loss: 0.7647 - val_accuracy: 0.6640\n",
            "Epoch 10/100\n",
            "47/47 - 0s - loss: 0.7045 - accuracy: 0.7000 - val_loss: 0.7563 - val_accuracy: 0.6800\n",
            "Epoch 11/100\n",
            "47/47 - 0s - loss: 0.6754 - accuracy: 0.7160 - val_loss: 0.7646 - val_accuracy: 0.6920\n",
            "Epoch 12/100\n",
            "47/47 - 0s - loss: 0.6810 - accuracy: 0.7013 - val_loss: 0.7454 - val_accuracy: 0.6660\n",
            "Epoch 13/100\n",
            "47/47 - 0s - loss: 0.6581 - accuracy: 0.7133 - val_loss: 0.8056 - val_accuracy: 0.6380\n",
            "Epoch 14/100\n",
            "47/47 - 0s - loss: 0.6536 - accuracy: 0.7153 - val_loss: 0.7674 - val_accuracy: 0.6680\n",
            "Epoch 15/100\n",
            "47/47 - 0s - loss: 0.6404 - accuracy: 0.7260 - val_loss: 0.7422 - val_accuracy: 0.6880\n",
            "Epoch 16/100\n",
            "47/47 - 0s - loss: 0.6388 - accuracy: 0.7227 - val_loss: 0.7847 - val_accuracy: 0.6780\n",
            "Epoch 17/100\n",
            "47/47 - 0s - loss: 0.6383 - accuracy: 0.7213 - val_loss: 0.7446 - val_accuracy: 0.6960\n",
            "Epoch 18/100\n",
            "47/47 - 0s - loss: 0.6267 - accuracy: 0.7333 - val_loss: 0.7363 - val_accuracy: 0.6980\n",
            "Epoch 19/100\n",
            "47/47 - 0s - loss: 0.6170 - accuracy: 0.7320 - val_loss: 0.7311 - val_accuracy: 0.7000\n",
            "Epoch 20/100\n",
            "47/47 - 0s - loss: 0.6104 - accuracy: 0.7420 - val_loss: 0.7532 - val_accuracy: 0.6860\n",
            "Epoch 21/100\n",
            "47/47 - 0s - loss: 0.6053 - accuracy: 0.7327 - val_loss: 0.7415 - val_accuracy: 0.6980\n",
            "Epoch 22/100\n",
            "47/47 - 0s - loss: 0.6165 - accuracy: 0.7360 - val_loss: 0.7352 - val_accuracy: 0.6940\n",
            "Epoch 23/100\n",
            "47/47 - 0s - loss: 0.5909 - accuracy: 0.7473 - val_loss: 0.7349 - val_accuracy: 0.7040\n",
            "Epoch 24/100\n",
            "Restoring model weights from the end of the best epoch.\n",
            "47/47 - 0s - loss: 0.5822 - accuracy: 0.7440 - val_loss: 0.7368 - val_accuracy: 0.6980\n",
            "Epoch 00024: early stopping\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f049f67dfd0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pQIArNvPCBBs"
      },
      "source": [
        "pred = model.predict(x_test)\n",
        "pred = np.argmax(pred,axis=1) \n",
        "# raw probabilities to chosen class (highest probability)\n"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uxDEddWGCPvQ",
        "outputId": "37ab1769-a466-40e7-8942-65eb14cb7c19",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from sklearn import metrics\n",
        "\n",
        "y_compare = np.argmax(y_test,axis=1) \n",
        "score = metrics.accuracy_score(y_compare, pred)\n",
        "print(\"Accuracy score: {}\".format(score))"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy score: 0.7\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "znnHibwVCqdA"
      },
      "source": [
        "Calculate Classification Log Loss"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d2rmZUoRCbRB",
        "outputId": "32860ed1-b265-40f6-8602-ba9301ba5050",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "from IPython.display import display\n",
        "\n",
        "# Don't display numpy in scientific notation\n",
        "np.set_printoptions(precision=4)\n",
        "np.set_printoptions(suppress=True)\n",
        "\n",
        "# Generate predictions\n",
        "pred = model.predict(x_test)\n",
        "\n",
        "print(\"Numpy array of predictions\")\n",
        "display(pred[0:5])\n",
        "\n",
        "print(\"As percent probability\")\n",
        "print(pred[0]*100)\n",
        "\n",
        "score = metrics.log_loss(y_test, pred)\n",
        "print(\"Log loss score: {}\".format(score))\n",
        "\n",
        "# raw probabilities to chosen class (highest probability)\n",
        "pred = np.argmax(pred,axis=1)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Numpy array of predictions\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "array([[0.    , 0.0798, 0.6945, 0.2249, 0.0008, 0.    , 0.    ],\n",
              "       [0.    , 0.8257, 0.1734, 0.    , 0.0009, 0.    , 0.    ],\n",
              "       [0.    , 0.73  , 0.2671, 0.0001, 0.0027, 0.    , 0.    ],\n",
              "       [0.    , 0.3477, 0.6435, 0.0067, 0.0021, 0.    , 0.    ],\n",
              "       [0.    , 0.0235, 0.8095, 0.167 , 0.    , 0.    , 0.    ]],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "As percent probability\n",
            "[ 0.      7.9843 69.45   22.4858  0.0796  0.0003  0.    ]\n",
            "Log loss score: 0.7311158122420311\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9H0oU9LeC1ua",
        "outputId": "5765eb89-f07b-47d5-b8c4-9806c8bfb1f9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 367
        }
      },
      "source": [
        "import numpy as np\n",
        "from sklearn import svm, datasets\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import confusion_matrix\n",
        "\n",
        "# Compute confusion matrix\n",
        "cm = confusion_matrix(y_compare, pred)\n",
        "np.set_printoptions(precision=2)\n",
        "print('Confusion matrix, without normalization')\n",
        "print(cm)\n",
        "plt.figure()\n",
        "plot_confusion_matrix(cm, products)\n",
        "\n",
        "# Normalize the confusion matrix by row (i.e by the number of samples\n",
        "# in each class)\n",
        "cm_normalized = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
        "print('Normalized confusion matrix')\n",
        "print(cm_normalized)\n",
        "plt.figure()\n",
        "plot_confusion_matrix(cm_normalized, products, \n",
        "        title='Normalized confusion matrix')\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Confusion matrix, without normalization\n",
            "[[ 18   2   0   0   0   2   0]\n",
            " [  5 190  52   0   0   2   0]\n",
            " [  0  49 136   2   0   0   0]\n",
            " [  0   0   9   5   0   0   0]\n",
            " [  0   8   0   0   0   0   0]\n",
            " [  7   9   0   0   0   1   0]\n",
            " [  3   0   0   0   0   0   0]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-796bba5eb6d6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Confusion matrix, without normalization'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m \u001b[0mplot_confusion_matrix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproducts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'plt' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L0J5KsyWY-y2"
      },
      "source": [
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for product\n",
        "df = pd.concat([df,pd.get_dummies(df['product'],prefix=\"product\")],axis=1)\n",
        "df.drop('product', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('age').drop('id')\n",
        "x = df[x_columns].values\n",
        "y = df['age'].values"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eIa2YHneY-9F",
        "outputId": "09c451ed-4d41-4a4f-cd00-610550c9199a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "from sklearn import metrics\n",
        "from scipy.stats import zscore\n",
        "from sklearn.model_selection import KFold\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation\n",
        "# Cross-Validate\n",
        "kf = KFold(5, shuffle=True, random_state=42) # Use for KFold classification\n",
        "    \n",
        "oos_y = []\n",
        "oos_pred = []\n",
        "\n",
        "fold = 0\n",
        "for train, test in kf.split(x):\n",
        "    fold+=1\n",
        "    print(f\"Fold #{fold}\")\n",
        "        \n",
        "    x_train = x[train]\n",
        "    y_train = y[train]\n",
        "    x_test = x[test]\n",
        "    y_test = y[test]\n",
        "    \n",
        "    model = Sequential()\n",
        "    model.add(Dense(20, input_dim=x.shape[1], activation='relu'))\n",
        "    model.add(Dense(10, activation='relu'))\n",
        "    model.add(Dense(1))\n",
        "    model.compile(loss='mean_squared_error', optimizer='adam')\n",
        "    \n",
        "    model.fit(x_train,y_train,validation_data=(x_test,y_test),verbose=0,\n",
        "              epochs=500)\n",
        "    \n",
        "    pred = model.predict(x_test)\n",
        "    \n",
        "    oos_y.append(y_test)\n",
        "    oos_pred.append(pred)    \n",
        "\n",
        "    # Measure this fold's RMSE\n",
        "    score = np.sqrt(metrics.mean_squared_error(pred,y_test))\n",
        "    print(f\"Fold score (RMSE): {score}\")\n",
        "\n",
        "# Build the oos prediction list and calculate the error.\n",
        "oos_y = np.concatenate(oos_y)\n",
        "oos_pred = np.concatenate(oos_pred)\n",
        "score = np.sqrt(metrics.mean_squared_error(oos_pred,oos_y))\n",
        "print(f\"Final, out of sample score (RMSE): {score}\")    \n",
        "    \n",
        "# Write the cross-validated prediction\n",
        "oos_y = pd.DataFrame(oos_y)\n",
        "oos_pred = pd.DataFrame(oos_pred)\n",
        "oosDF = pd.concat( [df, oos_y, oos_pred],axis=1 )\n",
        "#oosDF.to_csv(filename_write,index=False)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fold #1\n",
            "Fold score (RMSE): 0.6156115062365571\n",
            "Fold #2\n",
            "Fold score (RMSE): 0.5203516860590484\n",
            "Fold #3\n",
            "Fold score (RMSE): 0.6284392047365168\n",
            "Fold #4\n",
            "Fold score (RMSE): 0.5657673438927229\n",
            "Fold #5\n",
            "Fold score (RMSE): 0.9333984798131855\n",
            "Final, out of sample score (RMSE): 0.668730834847354\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ntkYc4wRY_Dk"
      },
      "source": [
        ""
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kIB3set9pnr7"
      },
      "source": [
        "**Classification with Stratified K-Fold Cross-Validation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R_3LW-Fdpnfr"
      },
      "source": [
        ""
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "87JReYYKY_AV"
      },
      "source": [
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['age'] = zscore(df['age'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('product').drop('id')\n",
        "x = df[x_columns].values\n",
        "dummies = pd.get_dummies(df['product']) # Classification\n",
        "products = dummies.columns\n",
        "y = dummies.values\n"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4xKH4wi5Y-4N",
        "outputId": "0cded3a1-484d-46aa-febb-e7c878db6700",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation\n",
        "\n",
        "# np.argmax(pred,axis=1)\n",
        "# Cross-validate\n",
        "# Use for StratifiedKFold classification\n",
        "kf = StratifiedKFold(5, shuffle=True, random_state=42) \n",
        "    \n",
        "oos_y = []\n",
        "oos_pred = []\n",
        "fold = 0\n",
        "\n",
        "# Must specify y StratifiedKFold for\n",
        "for train, test in kf.split(x,df['product']):  \n",
        "    fold+=1\n",
        "    print(f\"Fold #{fold}\")\n",
        "        \n",
        "    x_train = x[train]\n",
        "    y_train = y[train]\n",
        "    x_test = x[test]\n",
        "    y_test = y[test]\n",
        "    \n",
        "    model = Sequential()\n",
        "    model.add(Dense(50, input_dim=x.shape[1], activation='relu')) # Hidden 1\n",
        "    model.add(Dense(25, activation='relu')) # Hidden 2\n",
        "    model.add(Dense(y.shape[1],activation='softmax')) # Output\n",
        "    model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
        "\n",
        "    model.fit(x_train,y_train,validation_data=(x_test,y_test),verbose=0,\\\n",
        "              epochs=500)\n",
        "    \n",
        "    pred = model.predict(x_test)\n",
        "    \n",
        "    oos_y.append(y_test)\n",
        "    # raw probabilities to chosen class (highest probability)\n",
        "    pred = np.argmax(pred,axis=1) \n",
        "    oos_pred.append(pred)  \n",
        "\n",
        "    # Measure this fold's accuracy\n",
        "    y_compare = np.argmax(y_test,axis=1) # For accuracy calculation\n",
        "    score = metrics.accuracy_score(y_compare, pred)\n",
        "    print(f\"Fold score (accuracy): {score}\")\n",
        "\n",
        "# Build the oos prediction list and calculate the error.\n",
        "oos_y = np.concatenate(oos_y)\n",
        "oos_pred = np.concatenate(oos_pred)\n",
        "oos_y_compare = np.argmax(oos_y,axis=1) # For accuracy calculation\n",
        "\n",
        "score = metrics.accuracy_score(oos_y_compare, oos_pred)\n",
        "print(f\"Final score (accuracy): {score}\")    \n",
        "    \n",
        "# Write the cross-validated prediction\n",
        "oos_y = pd.DataFrame(oos_y)\n",
        "oos_pred = pd.DataFrame(oos_pred)\n",
        "oosDF = pd.concat( [df, oos_y, oos_pred],axis=1 )\n",
        "#oosDF.to_csv(filename_write,index=False)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fold #1\n",
            "Fold score (accuracy): 0.6125\n",
            "Fold #2\n",
            "Fold score (accuracy): 0.6575\n",
            "Fold #3\n",
            "Fold score (accuracy): 0.6975\n",
            "Fold #4\n",
            "Fold score (accuracy): 0.65\n",
            "Fold #5\n",
            "Fold score (accuracy): 0.6875\n",
            "Final score (accuracy): 0.661\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EBIFMQaZY_-o"
      },
      "source": [
        "\n",
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for product\n",
        "df = pd.concat([df,pd.get_dummies(df['product'],prefix=\"product\")],axis=1)\n",
        "df.drop('product', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('age').drop('id')\n",
        "x = df[x_columns].values\n",
        "y = df['age'].values"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z-QTt6U_ZAEf",
        "outputId": "c0d286dd-e750-4b5f-dd1e-9841523cc953",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "from sklearn import metrics\n",
        "from scipy.stats import zscore\n",
        "from sklearn.model_selection import KFold\n",
        "\n",
        "# Keep a 10% holdout\n",
        "x_main, x_holdout, y_main, y_holdout = train_test_split(    \n",
        "    x, y, test_size=0.10) \n",
        "\n",
        "\n",
        "# Cross-validate\n",
        "kf = KFold(5)\n",
        "    \n",
        "oos_y = []\n",
        "oos_pred = []\n",
        "fold = 0\n",
        "for train, test in kf.split(x_main):        \n",
        "    fold+=1\n",
        "    print(f\"Fold #{fold}\")\n",
        "        \n",
        "    x_train = x_main[train]\n",
        "    y_train = y_main[train]\n",
        "    x_test = x_main[test]\n",
        "    y_test = y_main[test]\n",
        "    \n",
        "    model = Sequential()\n",
        "    model.add(Dense(20, input_dim=x.shape[1], activation='relu'))\n",
        "    model.add(Dense(5, activation='relu'))\n",
        "    model.add(Dense(1))\n",
        "    model.compile(loss='mean_squared_error', optimizer='adam')\n",
        "    \n",
        "    model.fit(x_train,y_train,validation_data=(x_test,y_test),\n",
        "              verbose=0,epochs=500)\n",
        "    \n",
        "    pred = model.predict(x_test)\n",
        "    \n",
        "    oos_y.append(y_test)\n",
        "    oos_pred.append(pred) \n",
        "\n",
        "    # Measure accuracy\n",
        "    score = np.sqrt(metrics.mean_squared_error(pred,y_test))\n",
        "    print(f\"Fold score (RMSE): {score}\")\n",
        "\n",
        "\n",
        "# Build the oos prediction list and calculate the error.\n",
        "oos_y = np.concatenate(oos_y)\n",
        "oos_pred = np.concatenate(oos_pred)\n",
        "score = np.sqrt(metrics.mean_squared_error(oos_pred,oos_y))\n",
        "print()\n",
        "print(f\"Cross-validated score (RMSE): {score}\")    \n",
        "    \n",
        "# Write the cross-validated prediction (from the last neural network)\n",
        "holdout_pred = model.predict(x_holdout)\n",
        "\n",
        "score = np.sqrt(metrics.mean_squared_error(holdout_pred,y_holdout))\n",
        "print(f\"Holdout score (RMSE): {score}\")"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fold #1\n",
            "Fold score (RMSE): 0.4776277338477689\n",
            "Fold #2\n",
            "Fold score (RMSE): 0.6345354145167714\n",
            "Fold #3\n",
            "Fold score (RMSE): 1.0198941816453264\n",
            "Fold #4\n",
            "Fold score (RMSE): 0.7445086146633026\n",
            "Fold #5\n",
            "Fold score (RMSE): 0.6189400192106891\n",
            "\n",
            "Cross-validated score (RMSE): 0.7222641360138151\n",
            "Holdout score (RMSE): 0.5313114856041595\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uRZy1CeJuDPr"
      },
      "source": [
        "L1 and L2 Regularization to Decrease **Overfitting**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ygDuz3ISZAL0"
      },
      "source": [
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['age'] = zscore(df['age'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('product').drop('id')\n",
        "x = df[x_columns].values\n",
        "dummies = pd.get_dummies(df['product']) # Classification\n",
        "products = dummies.columns\n",
        "y = dummies.values"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kHuHmfbHZARZ",
        "outputId": "8eab6ff3-07e7-4083-aedc-88c932aee066",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 527
        }
      },
      "source": [
        "########################################\n",
        "# Keras with L1/L2 for Regression\n",
        "########################################\n",
        "\n",
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import KFold\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation\n",
        "from tensorflow.keras import regularizers\n",
        "\n",
        "# Cross-validate\n",
        "kf = KFold(5, shuffle=True, random_state=42)\n",
        "    \n",
        "oos_y = []\n",
        "oos_pred = []\n",
        "fold = 0\n",
        "\n",
        "for train, test in kf.split(x):\n",
        "    fold+=1\n",
        "    print(f\"Fold #{fold}\")\n",
        "        \n",
        "    x_train = x[train]\n",
        "    y_train = y[train]\n",
        "    x_test = x[test]\n",
        "    y_test = y[test]\n",
        "    \n",
        "    #kernel_regularizer=regularizers.l2(0.01),\n",
        "    \n",
        "    model = Sequential()\n",
        "    model.add(Dense(50, input_dim=x.shape[1], \n",
        "            activation='relu',\n",
        "             activity_regularizer=regularizers.l1(1e-4))) # Hidden 1\n",
        "    model.add(Dense(25, activation='relu', \n",
        "                    activity_regularizer=regularizers.l1(1e-4))) # Hidden 2\n",
        "    model.add(Dense(y.shape[1],activation='softmax')) # Output\n",
        "    model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
        "\n",
        "    model.fit(x_train,y_train,validation_data=(x_test,y_test),\n",
        "              verbose=0,epochs=500)\n",
        "    \n",
        "    pred = model.predict(x_test)\n",
        "    \n",
        "    oos_y.append(y_test)\n",
        "    # raw probabilities to chosen class (highest probability)\n",
        "    pred = np.argmax(pred,axis=1) \n",
        "    oos_pred.append(pred)        \n",
        "\n",
        "    # Measure this fold's accuracy\n",
        "    y_compare = np.argmax(y_test,axis=1) # For accuracy calculation\n",
        "    score = metrics.accuracy_score(y_compare, pred)\n",
        "    print(f\"Fold score (accuracy): {score}\")\n",
        "\n",
        "\n",
        "# Build the oos prediction list and calculate the error.\n",
        "oos_y = np.concatenate(oos_y)\n",
        "oos_pred = np.concatenate(oos_pred)\n",
        "oos_y_compare = np.argmax(oos_y,axis=1) # For accuracy calculation\n",
        "\n",
        "score = metrics.accuracy_score(oos_y_compare, oos_pred)\n",
        "print(f\"Final score (accuracy): {score}\")    \n",
        "    \n",
        "# Write the cross-validated prediction\n",
        "oos_y = pd.DataFrame(oos_y)\n",
        "oos_pred = pd.DataFrame(oos_pred)\n",
        "oosDF = pd.concat( [df, oos_y, oos_pred],axis=1 )\n",
        "#oosDF.to_csv(filename_write,index=False)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fold #1\n",
            "Fold score (accuracy): 0.6475\n",
            "Fold #2\n",
            "Fold score (accuracy): 0.685\n",
            "Fold #3\n",
            "Fold score (accuracy): 0.6975\n",
            "Fold #4\n",
            "Fold score (accuracy): 0.6375\n",
            "Fold #5\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-30-5ea467392a30>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m     model.fit(x_train,y_train,validation_data=(x_test,y_test),\n\u001b[0;32m---> 42\u001b[0;31m               verbose=0,epochs=500)\n\u001b[0m\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m     \u001b[0mpred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36m_method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    106\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_method_wrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_in_multi_worker_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    110\u001b[0m     \u001b[0;31m# Running inside `run_distribute_coordinator` already.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1096\u001b[0m                 batch_size=batch_size):\n\u001b[1;32m   1097\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1098\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1099\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1100\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    778\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    779\u001b[0m         \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 780\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    781\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    782\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    805\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    806\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 807\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    808\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    809\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2827\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2828\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2829\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2830\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2831\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1846\u001b[0m                            resource_variable_ops.BaseResourceVariable))],\n\u001b[1;32m   1847\u001b[0m         \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1848\u001b[0;31m         cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[1;32m   1849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1850\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1922\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1923\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1924\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1925\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1926\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    548\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XPIkfg1P1LLu"
      },
      "source": [
        "**Lets try to use dropout layers method to avoid overfitting the model.**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2tGUc3M8ZAUw"
      },
      "source": [
        "########################################\n",
        "# Keras with dropout for Classification\n",
        "########################################\n",
        "\n",
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import KFold\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation, Dropout\n",
        "from tensorflow.keras import regularizers\n",
        "\n",
        "# Cross-validate\n",
        "kf = KFold(5, shuffle=True, random_state=42)\n",
        "    \n",
        "oos_y = []\n",
        "oos_pred = []\n",
        "fold = 0\n",
        "\n",
        "for train, test in kf.split(x):\n",
        "    fold+=1\n",
        "    print(f\"Fold #{fold}\")\n",
        "        \n",
        "    x_train = x[train]\n",
        "    y_train = y[train]\n",
        "    x_test = x[test]\n",
        "    y_test = y[test]\n",
        "    \n",
        "    #kernel_regularizer=regularizers.l2(0.01),\n",
        "    \n",
        "    model = Sequential()\n",
        "    model.add(Dense(50, input_dim=x.shape[1], activation='relu')) # Hidden 1\n",
        "    model.add(Dropout(0.5))\n",
        "    model.add(Dense(25, activation='relu', \\\n",
        "                activity_regularizer=regularizers.l1(1e-4))) # Hidden 2\n",
        "    # Usually do not add dropout after final hidden layer\n",
        "    #model.add(Dropout(0.5)) \n",
        "    model.add(Dense(y.shape[1],activation='softmax')) # Output\n",
        "    model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
        "\n",
        "    model.fit(x_train,y_train,validation_data=(x_test,y_test),\\\n",
        "              verbose=0,epochs=500)\n",
        "    \n",
        "    pred = model.predict(x_test)\n",
        "    \n",
        "    oos_y.append(y_test)\n",
        "    # raw probabilities to chosen class (highest probability)\n",
        "    pred = np.argmax(pred,axis=1) \n",
        "    oos_pred.append(pred)        \n",
        "\n",
        "    # Measure this fold's accuracy\n",
        "    y_compare = np.argmax(y_test,axis=1) # For accuracy calculation\n",
        "    score = metrics.accuracy_score(y_compare, pred)\n",
        "    print(f\"Fold score (accuracy): {score}\")\n",
        "\n",
        "\n",
        "# Build the oos prediction list and calculate the error.\n",
        "oos_y = np.concatenate(oos_y)\n",
        "oos_pred = np.concatenate(oos_pred)\n",
        "oos_y_compare = np.argmax(oos_y,axis=1) # For accuracy calculation\n",
        "\n",
        "score = metrics.accuracy_score(oos_y_compare, oos_pred)\n",
        "print(f\"Final score (accuracy): {score}\")    \n",
        "    \n",
        "# Write the cross-validated prediction\n",
        "oos_y = pd.DataFrame(oos_y)\n",
        "oos_pred = pd.DataFrame(oos_pred)\n",
        "oosDF = pd.concat( [df, oos_y, oos_pred],axis=1 )\n",
        "#oosDF.to_csv(filename_write,index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HPgMuPSPQ-Ke"
      },
      "source": [
        "**Bootstrapping for Classification**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pjJczlsZZAOs"
      },
      "source": [
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['age'] = zscore(df['age'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('product').drop('id')\n",
        "x = df[x_columns].values\n",
        "dummies = pd.get_dummies(df['product']) # Classification\n",
        "products = dummies.columns\n",
        "y = dummies.values"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m5HbdZEoZAIp",
        "outputId": "4926ba19-d2cd-4ac6-8a20-047631fb8079",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "import time\n",
        "import statistics\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation\n",
        "from tensorflow.keras import regularizers\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from sklearn.model_selection import StratifiedShuffleSplit\n",
        "\n",
        "SPLITS = 50\n",
        "\n",
        "# Bootstrap\n",
        "boot = StratifiedShuffleSplit(n_splits=SPLITS, test_size=0.1, \n",
        "                                random_state=42)\n",
        "\n",
        "# Track progress\n",
        "mean_benchmark = []\n",
        "epochs_needed = []\n",
        "num = 0\n",
        "\n",
        "# Loop through samples\n",
        "for train, test in boot.split(x,df['product']):\n",
        "    start_time = time.time()\n",
        "    num+=1\n",
        "\n",
        "    # Split train and test\n",
        "    x_train = x[train]\n",
        "    y_train = y[train]\n",
        "    x_test = x[test]\n",
        "    y_test = y[test]\n",
        "\n",
        "    # Construct neural network\n",
        "    model = Sequential()\n",
        "    model.add(Dense(50, input_dim=x.shape[1], activation='relu')) # Hidden 1\n",
        "    model.add(Dense(25, activation='relu')) # Hidden 2\n",
        "    model.add(Dense(y.shape[1],activation='softmax')) # Output\n",
        "    model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
        "    monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, \n",
        "        patience=25, verbose=0, mode='auto', restore_best_weights=True)\n",
        "\n",
        "    # Train on the bootstrap sample\n",
        "    model.fit(x_train,y_train,validation_data=(x_test,y_test),\n",
        "              callbacks=[monitor],verbose=0,epochs=1000)\n",
        "    epochs = monitor.stopped_epoch\n",
        "    epochs_needed.append(epochs)\n",
        "    \n",
        "    # Predict on the out of boot (validation)\n",
        "    pred = model.predict(x_test)\n",
        "  \n",
        "    # Measure this bootstrap's log loss\n",
        "    y_compare = np.argmax(y_test,axis=1) # For log loss calculation\n",
        "    score = metrics.log_loss(y_compare, pred)\n",
        "    mean_benchmark.append(score)\n",
        "    m1 = statistics.mean(mean_benchmark)\n",
        "    m2 = statistics.mean(epochs_needed)\n",
        "    mdev = statistics.pstdev(mean_benchmark)\n",
        "    \n",
        "    # Record this iteration\n",
        "    time_took = time.time() - start_time\n",
        "    \n",
        "    print(f\"#{num}: score={score:.6f}, mean score={m1:.6f}, stdev={mdev:.6f},epochs={epochs}, mean epochs={int(m2)}\")"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "#1: score=0.674332, mean score=0.674332, stdev=0.000000,epochs=63, mean epochs=63\n",
            "#2: score=0.655220, mean score=0.664776, stdev=0.009556,epochs=70, mean epochs=66\n",
            "#3: score=0.680094, mean score=0.669882, stdev=0.010631,epochs=49, mean epochs=60\n",
            "#4: score=0.666653, mean score=0.669074, stdev=0.009312,epochs=59, mean epochs=60\n",
            "#5: score=0.673500, mean score=0.669960, stdev=0.008515,epochs=41, mean epochs=56\n",
            "#6: score=0.693438, mean score=0.673873, stdev=0.011704,epochs=49, mean epochs=55\n",
            "#7: score=0.704520, mean score=0.678251, stdev=0.015246,epochs=59, mean epochs=55\n",
            "#8: score=0.746527, mean score=0.686785, stdev=0.026707,epochs=59, mean epochs=56\n",
            "#9: score=0.636930, mean score=0.681246, stdev=0.029656,epochs=57, mean epochs=56\n",
            "#10: score=0.673650, mean score=0.680486, stdev=0.028226,epochs=65, mean epochs=57\n",
            "#11: score=0.665460, mean score=0.679120, stdev=0.027257,epochs=78, mean epochs=59\n",
            "#12: score=0.711236, mean score=0.681796, stdev=0.027565,epochs=57, mean epochs=58\n",
            "#13: score=0.747032, mean score=0.686815, stdev=0.031679,epochs=52, mean epochs=58\n",
            "#14: score=0.720985, mean score=0.689255, stdev=0.031770,epochs=53, mean epochs=57\n",
            "#15: score=0.674882, mean score=0.688297, stdev=0.030901,epochs=55, mean epochs=57\n",
            "#16: score=0.754003, mean score=0.692404, stdev=0.033885,epochs=89, mean epochs=59\n",
            "#17: score=0.610770, mean score=0.687602, stdev=0.038073,epochs=88, mean epochs=61\n",
            "#18: score=0.639791, mean score=0.684946, stdev=0.038587,epochs=55, mean epochs=61\n",
            "#19: score=0.626052, mean score=0.681846, stdev=0.039794,epochs=44, mean epochs=60\n",
            "#20: score=0.699238, mean score=0.682716, stdev=0.038971,epochs=54, mean epochs=59\n",
            "#21: score=0.624818, mean score=0.679958, stdev=0.039981,epochs=54, mean epochs=59\n",
            "#22: score=0.716855, mean score=0.681636, stdev=0.039810,epochs=46, mean epochs=58\n",
            "#23: score=0.612518, mean score=0.678630, stdev=0.041408,epochs=65, mean epochs=59\n",
            "#24: score=0.705195, mean score=0.679737, stdev=0.040882,epochs=66, mean epochs=59\n",
            "#25: score=0.549870, mean score=0.674543, stdev=0.047457,epochs=83, mean epochs=60\n",
            "#26: score=0.665452, mean score=0.674193, stdev=0.046568,epochs=56, mean epochs=60\n",
            "#27: score=0.695088, mean score=0.674967, stdev=0.045868,epochs=81, mean epochs=61\n",
            "#28: score=0.723186, mean score=0.676689, stdev=0.045921,epochs=43, mean epochs=60\n",
            "#29: score=0.664861, mean score=0.676281, stdev=0.045174,epochs=70, mean epochs=60\n",
            "#30: score=0.705137, mean score=0.677243, stdev=0.044716,epochs=59, mean epochs=60\n",
            "#31: score=0.705073, mean score=0.678141, stdev=0.044263,epochs=68, mean epochs=60\n",
            "#32: score=0.705728, mean score=0.679003, stdev=0.043829,epochs=55, mean epochs=60\n",
            "#33: score=0.627971, mean score=0.677456, stdev=0.044038,epochs=73, mean epochs=61\n",
            "#34: score=0.610989, mean score=0.675502, stdev=0.044815,epochs=55, mean epochs=60\n",
            "#35: score=0.519729, mean score=0.671051, stdev=0.051230,epochs=81, mean epochs=61\n",
            "#36: score=0.656488, mean score=0.670646, stdev=0.050570,epochs=55, mean epochs=61\n",
            "#37: score=0.693616, mean score=0.671267, stdev=0.050021,epochs=55, mean epochs=61\n",
            "#38: score=0.686041, mean score=0.671656, stdev=0.049415,epochs=60, mean epochs=61\n",
            "#39: score=0.764597, mean score=0.674039, stdev=0.050941,epochs=57, mean epochs=60\n",
            "#40: score=0.695846, mean score=0.674584, stdev=0.050416,epochs=63, mean epochs=61\n",
            "#41: score=0.733243, mean score=0.676015, stdev=0.050613,epochs=37, mean epochs=60\n",
            "#42: score=0.664527, mean score=0.675741, stdev=0.050037,epochs=52, mean epochs=60\n",
            "#43: score=0.696210, mean score=0.676217, stdev=0.049548,epochs=47, mean epochs=59\n",
            "#44: score=0.753959, mean score=0.677984, stdev=0.050333,epochs=68, mean epochs=60\n",
            "#45: score=0.619045, mean score=0.676675, stdev=0.050523,epochs=85, mean epochs=60\n",
            "#46: score=0.682834, mean score=0.676808, stdev=0.049979,epochs=72, mean epochs=60\n",
            "#47: score=0.712672, mean score=0.677571, stdev=0.049715,epochs=59, mean epochs=60\n",
            "#48: score=0.740247, mean score=0.678877, stdev=0.050002,epochs=86, mean epochs=61\n",
            "#49: score=0.635310, mean score=0.677988, stdev=0.049871,epochs=105, mean epochs=62\n",
            "#50: score=0.676204, mean score=0.677952, stdev=0.049371,epochs=46, mean epochs=61\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y40KdNbkRk4Z"
      },
      "source": [
        "**`Benchmarking`**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A_HirRLfZAB5"
      },
      "source": [
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],\n",
        "               axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['age'] = zscore(df['age'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('product').drop('id')\n",
        "x = df[x_columns].values\n",
        "dummies = pd.get_dummies(df['product']) # Classification\n",
        "products = dummies.columns\n",
        "y = dummies.values"
      ],
      "execution_count": 54,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xO9fiX-CY_7d",
        "outputId": "1357e361-9b4b-45cb-c7c2-9b1c5b2a42e9",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "import time\n",
        "import tensorflow.keras.initializers\n",
        "import statistics\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation, Dropout\n",
        "from tensorflow.keras import regularizers\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from sklearn.model_selection import StratifiedShuffleSplit\n",
        "from tensorflow.keras.layers import LeakyReLU,PReLU\n",
        "\n",
        "SPLITS = 100\n",
        "\n",
        "# Bootstrap\n",
        "boot = StratifiedShuffleSplit(n_splits=SPLITS, test_size=0.1)\n",
        "\n",
        "# Track progress\n",
        "mean_benchmark = []\n",
        "epochs_needed = []\n",
        "num = 0\n",
        "\n",
        "# Loop through samples\n",
        "for train, test in boot.split(x,df['product']):\n",
        "    start_time = time.time()\n",
        "    num+=1\n",
        "\n",
        "    # Split train and test\n",
        "    x_train = x[train]\n",
        "    y_train = y[train]\n",
        "    x_test = x[test]\n",
        "    y_test = y[test]\n",
        "\n",
        "    # Construct neural network\n",
        "    model = Sequential()\n",
        "    model.add(Dense(100, input_dim=x.shape[1], activation=PReLU(), \\\n",
        "        kernel_regularizer=regularizers.l2(1e-4))) # Hidden 1\n",
        "    model.add(Dropout(0.5))\n",
        "    model.add(Dense(100, activation=PReLU(), \\\n",
        "        activity_regularizer=regularizers.l2(1e-4))) # Hidden 2\n",
        "    model.add(Dropout(0.5))\n",
        "    model.add(Dense(100, activation=PReLU(), \\\n",
        "        activity_regularizer=regularizers.l2(1e-4)\n",
        "    )) # Hidden 3\n",
        "#    model.add(Dropout(0.5)) - Usually better performance \n",
        "# without dropout on final layer\n",
        "    model.add(Dense(y.shape[1],activation='softmax')) # Output\n",
        "    model.compile(loss='categorical_crossentropy', optimizer='adam')\n",
        "    monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, \n",
        "        patience=100, verbose=0, mode='auto', restore_best_weights=True)\n",
        "\n",
        "    # Train on the bootstrap sample\n",
        "    model.fit(x_train,y_train,validation_data=(x_test,y_test), \\\n",
        "              callbacks=[monitor],verbose=0,epochs=1000)\n",
        "    epochs = monitor.stopped_epoch\n",
        "    epochs_needed.append(epochs)\n",
        "    \n",
        "    # Predict on the out of boot (validation)\n",
        "    pred = model.predict(x_test)\n",
        "  \n",
        "    # Measure this bootstrap's log loss\n",
        "    y_compare = np.argmax(y_test,axis=1) # For log loss calculation\n",
        "    score = metrics.log_loss(y_compare, pred)\n",
        "    mean_benchmark.append(score)\n",
        "    m1 = statistics.mean(mean_benchmark)\n",
        "    m2 = statistics.mean(epochs_needed)\n",
        "    mdev = statistics.pstdev(mean_benchmark)\n",
        "    \n",
        "\n",
        "    # Record this iteration\n",
        "    time_took = time.time() - start_time\n",
        "    \n",
        "    print(f\"#{num}: score={score:.6f}, mean score={m1:.6f}, stdev={mdev:.6f},epochs={epochs}, mean epochs={int(m2)}\")"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "#1: score=0.571549, mean score=0.571549, stdev=0.000000,epochs=168, mean epochs=168\n",
            "#2: score=0.649841, mean score=0.610695, stdev=0.039146,epochs=167, mean epochs=167\n",
            "#3: score=0.596874, mean score=0.606088, stdev=0.032620,epochs=275, mean epochs=203\n",
            "#4: score=0.652432, mean score=0.617674, stdev=0.034652,epochs=160, mean epochs=192\n",
            "#5: score=0.615379, mean score=0.617215, stdev=0.031007,epochs=156, mean epochs=185\n",
            "#6: score=0.615250, mean score=0.616888, stdev=0.028315,epochs=170, mean epochs=182\n",
            "#7: score=0.757583, mean score=0.636987, stdev=0.055777,epochs=150, mean epochs=178\n",
            "#8: score=0.571258, mean score=0.628771, stdev=0.056522,epochs=226, mean epochs=184\n",
            "#9: score=0.627799, mean score=0.628663, stdev=0.053290,epochs=189, mean epochs=184\n",
            "#10: score=0.689685, mean score=0.634765, stdev=0.053768,epochs=183, mean epochs=184\n",
            "#11: score=0.641139, mean score=0.635345, stdev=0.051299,epochs=178, mean epochs=183\n",
            "#12: score=0.605789, mean score=0.632882, stdev=0.049789,epochs=212, mean epochs=186\n",
            "#13: score=0.672999, mean score=0.635967, stdev=0.049016,epochs=154, mean epochs=183\n",
            "#14: score=0.692839, mean score=0.640030, stdev=0.049452,epochs=181, mean epochs=183\n",
            "#15: score=0.564058, mean score=0.634965, stdev=0.051396,epochs=266, mean epochs=189\n",
            "#16: score=0.616042, mean score=0.633782, stdev=0.049975,epochs=321, mean epochs=197\n",
            "#17: score=0.700583, mean score=0.637712, stdev=0.050967,epochs=177, mean epochs=196\n",
            "#18: score=0.660753, mean score=0.638992, stdev=0.049811,epochs=173, mean epochs=194\n",
            "#19: score=0.597866, mean score=0.636827, stdev=0.049345,epochs=180, mean epochs=194\n",
            "#20: score=0.561256, mean score=0.633049, stdev=0.050837,epochs=389, mean epochs=203\n",
            "#21: score=0.693610, mean score=0.635933, stdev=0.051261,epochs=173, mean epochs=202\n",
            "#22: score=0.593336, mean score=0.633996, stdev=0.050862,epochs=199, mean epochs=202\n",
            "#23: score=0.615629, mean score=0.633198, stdev=0.049885,epochs=175, mean epochs=200\n",
            "#24: score=0.605518, mean score=0.632045, stdev=0.049147,epochs=178, mean epochs=200\n",
            "#25: score=0.650814, mean score=0.632795, stdev=0.048294,epochs=235, mean epochs=201\n",
            "#26: score=0.552135, mean score=0.629693, stdev=0.049832,epochs=262, mean epochs=203\n",
            "#27: score=0.611696, mean score=0.629026, stdev=0.049019,epochs=206, mean epochs=203\n",
            "#28: score=0.681984, mean score=0.630918, stdev=0.049128,epochs=182, mean epochs=203\n",
            "#29: score=0.641665, mean score=0.631288, stdev=0.048314,epochs=167, mean epochs=201\n",
            "#30: score=0.594208, mean score=0.630052, stdev=0.047966,epochs=172, mean epochs=200\n",
            "#31: score=0.592537, mean score=0.628842, stdev=0.047649,epochs=198, mean epochs=200\n",
            "#32: score=0.583101, mean score=0.627413, stdev=0.047569,epochs=173, mean epochs=199\n",
            "#33: score=0.605593, mean score=0.626752, stdev=0.046992,epochs=233, mean epochs=200\n",
            "#34: score=0.663130, mean score=0.627821, stdev=0.046702,epochs=168, mean epochs=199\n",
            "#35: score=0.638073, mean score=0.628114, stdev=0.046062,epochs=180, mean epochs=199\n",
            "#36: score=0.700713, mean score=0.630131, stdev=0.046958,epochs=170, mean epochs=198\n",
            "#37: score=0.604718, mean score=0.629444, stdev=0.046502,epochs=161, mean epochs=197\n",
            "#38: score=0.661927, mean score=0.630299, stdev=0.046180,epochs=224, mean epochs=198\n",
            "#39: score=0.704723, mean score=0.632207, stdev=0.047077,epochs=134, mean epochs=196\n",
            "#40: score=0.609649, mean score=0.631643, stdev=0.046618,epochs=160, mean epochs=195\n",
            "#41: score=0.622421, mean score=0.631418, stdev=0.046068,epochs=239, mean epochs=196\n",
            "#42: score=0.593326, mean score=0.630511, stdev=0.045886,epochs=192, mean epochs=196\n",
            "#43: score=0.727248, mean score=0.632761, stdev=0.047635,epochs=114, mean epochs=194\n",
            "#44: score=0.631161, mean score=0.632725, stdev=0.047091,epochs=246, mean epochs=195\n",
            "#45: score=0.689708, mean score=0.633991, stdev=0.047317,epochs=190, mean epochs=195\n",
            "#46: score=0.620189, mean score=0.633691, stdev=0.046843,epochs=222, mean epochs=196\n",
            "#47: score=0.664499, mean score=0.634347, stdev=0.046554,epochs=164, mean epochs=195\n",
            "#48: score=0.683623, mean score=0.635373, stdev=0.046601,epochs=139, mean epochs=194\n",
            "#49: score=0.696430, mean score=0.636619, stdev=0.046924,epochs=145, mean epochs=193\n",
            "#50: score=0.629676, mean score=0.636480, stdev=0.046463,epochs=168, mean epochs=192\n",
            "#51: score=0.580769, mean score=0.635388, stdev=0.046649,epochs=273, mean epochs=194\n",
            "#52: score=0.723727, mean score=0.637087, stdev=0.047765,epochs=130, mean epochs=193\n",
            "#53: score=0.628013, mean score=0.636916, stdev=0.047328,epochs=214, mean epochs=193\n",
            "#54: score=0.595266, mean score=0.636144, stdev=0.047223,epochs=236, mean epochs=194\n",
            "#55: score=0.595530, mean score=0.635406, stdev=0.047105,epochs=173, mean epochs=194\n",
            "#56: score=0.648560, mean score=0.635641, stdev=0.046715,epochs=234, mean epochs=194\n",
            "#57: score=0.674720, mean score=0.636326, stdev=0.046587,epochs=165, mean epochs=194\n",
            "#58: score=0.724767, mean score=0.637851, stdev=0.047597,epochs=176, mean epochs=193\n",
            "#59: score=0.683495, mean score=0.638625, stdev=0.047558,epochs=180, mean epochs=193\n",
            "#60: score=0.583810, mean score=0.637711, stdev=0.047679,epochs=233, mean epochs=194\n",
            "#61: score=0.606360, mean score=0.637197, stdev=0.047454,epochs=241, mean epochs=195\n",
            "#62: score=0.621469, mean score=0.636944, stdev=0.047112,epochs=177, mean epochs=194\n",
            "#63: score=0.638208, mean score=0.636964, stdev=0.046737,epochs=302, mean epochs=196\n",
            "#64: score=0.510427, mean score=0.634986, stdev=0.048954,epochs=283, mean epochs=197\n",
            "#65: score=0.763901, mean score=0.636970, stdev=0.051101,epochs=185, mean epochs=197\n",
            "#66: score=0.628557, mean score=0.636842, stdev=0.050723,epochs=265, mean epochs=198\n",
            "#67: score=0.585958, mean score=0.636083, stdev=0.050720,epochs=281, mean epochs=199\n",
            "#68: score=0.630569, mean score=0.636002, stdev=0.050350,epochs=166, mean epochs=199\n",
            "#69: score=0.619568, mean score=0.635764, stdev=0.050022,epochs=174, mean epochs=199\n",
            "#70: score=0.659204, mean score=0.636098, stdev=0.049741,epochs=176, mean epochs=198\n",
            "#71: score=0.678373, mean score=0.636694, stdev=0.049640,epochs=161, mean epochs=198\n",
            "#72: score=0.681725, mean score=0.637319, stdev=0.049575,epochs=170, mean epochs=197\n",
            "#73: score=0.656665, mean score=0.637584, stdev=0.049286,epochs=187, mean epochs=197\n",
            "#74: score=0.677788, mean score=0.638128, stdev=0.049171,epochs=143, mean epochs=196\n",
            "#75: score=0.659261, mean score=0.638409, stdev=0.048903,epochs=187, mean epochs=196\n",
            "#76: score=0.611072, mean score=0.638050, stdev=0.048680,epochs=176, mean epochs=196\n",
            "#77: score=0.647854, mean score=0.638177, stdev=0.048375,epochs=242, mean epochs=197\n",
            "#78: score=0.658219, mean score=0.638434, stdev=0.048117,epochs=157, mean epochs=196\n",
            "#79: score=0.678466, mean score=0.638941, stdev=0.048020,epochs=179, mean epochs=196\n",
            "#80: score=0.701835, mean score=0.639727, stdev=0.048228,epochs=131, mean epochs=195\n",
            "#81: score=0.613600, mean score=0.639404, stdev=0.048016,epochs=241, mean epochs=196\n",
            "#82: score=0.622269, mean score=0.639195, stdev=0.047760,epochs=239, mean epochs=196\n",
            "#83: score=0.706217, mean score=0.640003, stdev=0.048031,epochs=162, mean epochs=196\n",
            "#84: score=0.644837, mean score=0.640060, stdev=0.047747,epochs=194, mean epochs=196\n",
            "#85: score=0.600049, mean score=0.639590, stdev=0.047661,epochs=226, mean epochs=196\n",
            "#86: score=0.607683, mean score=0.639219, stdev=0.047507,epochs=211, mean epochs=196\n",
            "#87: score=0.638281, mean score=0.639208, stdev=0.047233,epochs=184, mean epochs=196\n",
            "#88: score=0.667426, mean score=0.639529, stdev=0.047059,epochs=180, mean epochs=196\n",
            "#89: score=0.666734, mean score=0.639834, stdev=0.046882,epochs=236, mean epochs=196\n",
            "#90: score=0.611721, mean score=0.639522, stdev=0.046713,epochs=199, mean epochs=196\n",
            "#91: score=0.658010, mean score=0.639725, stdev=0.046496,epochs=192, mean epochs=196\n",
            "#92: score=0.624827, mean score=0.639563, stdev=0.046268,epochs=206, mean epochs=196\n",
            "#93: score=0.756395, mean score=0.640819, stdev=0.047570,epochs=132, mean epochs=196\n",
            "#94: score=0.620396, mean score=0.640602, stdev=0.047363,epochs=182, mean epochs=196\n",
            "#95: score=0.710654, mean score=0.641339, stdev=0.047652,epochs=188, mean epochs=195\n",
            "#96: score=0.710242, mean score=0.642057, stdev=0.047917,epochs=158, mean epochs=195\n",
            "#97: score=0.665352, mean score=0.642297, stdev=0.047727,epochs=172, mean epochs=195\n",
            "#98: score=0.579723, mean score=0.641659, stdev=0.047898,epochs=352, mean epochs=196\n",
            "#99: score=0.695755, mean score=0.642205, stdev=0.047961,epochs=152, mean epochs=196\n",
            "#100: score=0.628913, mean score=0.642072, stdev=0.047739,epochs=192, mean epochs=196\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hTbs05Wt1Itr"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Udz3Nfvz1Iyz"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O1RIqp9I1I4R"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2rRz-1lR1I9s"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hv9gG9qm1JDN"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UteRIHXg1JHH"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NnwOX72Y1I2X"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G8qywNu51Iwg"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "23Cl8vlkE18T"
      },
      "source": [
        "Lets try some regression "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o9XObylODmew"
      },
      "source": [
        "\n",
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "from sklearn.model_selection import train_test_split\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for product\n",
        "df = pd.concat([df,pd.get_dummies(df['product'],prefix=\"product\")],axis=1)\n",
        "df.drop('product', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('age').drop('id')\n",
        "x = df[x_columns].values\n",
        "y = df['age'].values\n",
        "\n",
        "# Create train/test\n",
        "x_train, x_test, y_train, y_test = train_test_split(    \n",
        "    x, y, test_size=0.25, random_state=42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wCSFwhSBFDFw"
      },
      "source": [
        "\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "\n",
        "# Build the neural network\n",
        "model = Sequential()\n",
        "model.add(Dense(25, input_dim=x.shape[1], activation='relu')) # Hidden 1\n",
        "model.add(Dense(10, activation='relu')) # Hidden 2\n",
        "model.add(Dense(1)) # Output\n",
        "model.compile(loss='mean_squared_error', optimizer='adam')\n",
        "monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, \n",
        "                        patience=5, verbose=1, mode='auto', \n",
        "                        restore_best_weights=True)\n",
        "model.fit(x_train,y_train,validation_data=(x_test,y_test),\n",
        "          callbacks=[monitor],verbose=2,epochs=1000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DLT9j1W1FNu5"
      },
      "source": [
        "Mean Square Error"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m9Umr4seFIc_"
      },
      "source": [
        "\n",
        "from sklearn import metrics\n",
        "\n",
        "# Predict\n",
        "pred = model.predict(x_test)\n",
        "\n",
        "# Measure MSE error.  \n",
        "score = metrics.mean_squared_error(pred,y_test)\n",
        "print(\"Final score (MSE): {}\".format(score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bvsOmSjbFTup"
      },
      "source": [
        "\n",
        "Root Mean Square Error"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wSbJ5HzGFRsv"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "# Measure RMSE error.  RMSE is common for regression.\n",
        "score = np.sqrt(metrics.mean_squared_error(pred,y_test))\n",
        "print(\"Final score (RMSE): {}\".format(score))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B___M5gMFdjS"
      },
      "source": [
        "\n",
        "Lift Chart\n",
        "To generate a lift chart, perform the following activities:\n",
        "\n",
        "Sort the data by expected output. Plot the blue line above.\n",
        "For every point on the x-axis plot the predicted value for that same data point. This is the green line above.\n",
        "The x-axis is just 0 to 100% of the dataset. The expected always starts low and ends high.\n",
        "The y-axis is ranged according to the values predicted.\n",
        "Reading a lift chart:\n",
        "\n",
        "The expected and predict lines should be close. Notice where one is above the ot other.\n",
        "The below chart is the most accurate on lower age"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7BhXq09dFXDo"
      },
      "source": [
        "def chart_regression(pred, y, sort=True):\n",
        "    t = pd.DataFrame({'pred': pred, 'y': y.flatten()})\n",
        "    if sort:\n",
        "        t.sort_values(by=['y'], inplace=True)\n",
        "    plt.plot(t['y'].tolist(), label='expected')\n",
        "    plt.plot(t['pred'].tolist(), label='prediction')\n",
        "    plt.ylabel('output')\n",
        "    plt.legend()\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XE6XYM-WFl0b"
      },
      "source": [
        "\n",
        "chart_regression(pred.flatten(),y_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FefxMDKIZHpd"
      },
      "source": [
        "**Out-of-Sample Regression Predictions with K-Fold Cross-Validation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WsIKcoxMFnbG"
      },
      "source": [
        "\n",
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for product\n",
        "df = pd.concat([df,pd.get_dummies(df['product'],prefix=\"product\")],axis=1)\n",
        "df.drop('product', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('age').drop('id')\n",
        "x = df[x_columns].values\n",
        "y = df['age'].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mx3e2KUIZT0K"
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "from sklearn import metrics\n",
        "from scipy.stats import zscore \n",
        "from sklearn.model_selection import KFold\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.layers import Dense , Activation \n",
        "\n",
        "kf= KFold (5, shuffle = True, random_state=42)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tHYkX5fdIu2T"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "51xdzVwxIunE"
      },
      "source": [
        "**Bootstrapping for Regression**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PSwbX4sAIy15"
      },
      "source": [
        "\n",
        "import pandas as pd\n",
        "from scipy.stats import zscore\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Read the data set\n",
        "df = pd.read_csv(\n",
        "    \"https://data.heatonresearch.com/data/t81-558/jh-simple-dataset.csv\",\n",
        "    na_values=['NA','?'])\n",
        "\n",
        "# Generate dummies for job\n",
        "df = pd.concat([df,pd.get_dummies(df['job'],prefix=\"job\")],axis=1)\n",
        "df.drop('job', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for area\n",
        "df = pd.concat([df,pd.get_dummies(df['area'],prefix=\"area\")],axis=1)\n",
        "df.drop('area', axis=1, inplace=True)\n",
        "\n",
        "# Generate dummies for product\n",
        "df = pd.concat([df,pd.get_dummies(df['product'],prefix=\"product\")],axis=1)\n",
        "df.drop('product', axis=1, inplace=True)\n",
        "\n",
        "# Missing values for income\n",
        "med = df['income'].median()\n",
        "df['income'] = df['income'].fillna(med)\n",
        "\n",
        "# Standardize ranges\n",
        "df['income'] = zscore(df['income'])\n",
        "df['aspect'] = zscore(df['aspect'])\n",
        "df['save_rate'] = zscore(df['save_rate'])\n",
        "df['subscriptions'] = zscore(df['subscriptions'])\n",
        "\n",
        "# Convert to numpy - Classification\n",
        "x_columns = df.columns.drop('age').drop('id')\n",
        "x = df[x_columns].values\n",
        "y = df['age'].values"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QNqkHbxcI4KF",
        "outputId": "51fb59c9-2573-4d39-bd76-f90f35f82507",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "import numpy as np\n",
        "import time\n",
        "import statistics\n",
        "from sklearn import metrics\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Activation\n",
        "from tensorflow.keras import regularizers\n",
        "from tensorflow.keras.callbacks import EarlyStopping\n",
        "from sklearn.model_selection import ShuffleSplit\n",
        "\n",
        "SPLITS = 50\n",
        "\n",
        "# Bootstrap\n",
        "boot = ShuffleSplit(n_splits=SPLITS, test_size=0.1, random_state=42)\n",
        "\n",
        "# Track progress\n",
        "mean_benchmark = []\n",
        "epochs_needed = []\n",
        "num = 0\n",
        "\n",
        "# Loop through samples\n",
        "for train, test in boot.split(x):\n",
        "    start_time = time.time()\n",
        "    num+=1\n",
        "\n",
        "    # Split train and test\n",
        "    x_train = x[train]\n",
        "    y_train = y[train]\n",
        "    x_test = x[test]\n",
        "    y_test = y[test]\n",
        "\n",
        "    # Construct neural network\n",
        "    model = Sequential()\n",
        "    model.add(Dense(20, input_dim=x_train.shape[1], activation='relu'))\n",
        "    model.add(Dense(10, activation='relu'))\n",
        "    model.add(Dense(1))\n",
        "    model.compile(loss='mean_squared_error', optimizer='adam')\n",
        "    \n",
        "    monitor = EarlyStopping(monitor='val_loss', min_delta=1e-3, \n",
        "        patience=5, verbose=0, mode='auto', restore_best_weights=True)\n",
        "\n",
        "    # Train on the bootstrap sample\n",
        "    model.fit(x_train,y_train,validation_data=(x_test,y_test),\n",
        "              callbacks=[monitor],verbose=0,epochs=1000)\n",
        "    epochs = monitor.stopped_epoch\n",
        "    epochs_needed.append(epochs)\n",
        "    \n",
        "    # Predict on the out of boot (validation)\n",
        "    pred = model.predict(x_test)\n",
        "  \n",
        "    # Measure this bootstrap's log loss\n",
        "    score = np.sqrt(metrics.mean_squared_error(pred,y_test))\n",
        "    mean_benchmark.append(score)\n",
        "    m1 = statistics.mean(mean_benchmark)\n",
        "    m2 = statistics.mean(epochs_needed)\n",
        "    mdev = statistics.pstdev(mean_benchmark)\n",
        "    \n",
        "    # Record this iteration\n",
        "    time_took = time.time() - start_time\n",
        "    print(f\"#{num}: score={score:.6f}, mean score={m1:.6f}, stdev={mdev:.6f},epochs={epochs}, mean epochs={int(m2)}\")"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "#1: score=0.808796, mean score=0.808796, stdev=0.000000,epochs=94, mean epochs=94\n",
            "#2: score=1.074694, mean score=0.941745, stdev=0.132949,epochs=119, mean epochs=106\n",
            "#3: score=0.834188, mean score=0.905893, stdev=0.119810,epochs=116, mean epochs=109\n",
            "#4: score=0.575763, mean score=0.823360, stdev=0.176637,epochs=111, mean epochs=110\n",
            "#5: score=0.834616, mean score=0.825611, stdev=0.158053,epochs=95, mean epochs=107\n",
            "#6: score=0.792494, mean score=0.820092, stdev=0.144809,epochs=109, mean epochs=107\n",
            "#7: score=0.510901, mean score=0.775922, stdev=0.172279,epochs=114, mean epochs=108\n",
            "#8: score=0.523566, mean score=0.744377, stdev=0.181481,epochs=93, mean epochs=106\n",
            "#9: score=0.649852, mean score=0.733874, stdev=0.173662,epochs=113, mean epochs=107\n",
            "#10: score=1.208885, mean score=0.781375, stdev=0.217829,epochs=94, mean epochs=105\n",
            "#11: score=1.066917, mean score=0.807334, stdev=0.223326,epochs=91, mean epochs=104\n",
            "#12: score=0.834949, mean score=0.809635, stdev=0.213954,epochs=113, mean epochs=105\n",
            "#13: score=0.639029, mean score=0.796511, stdev=0.210528,epochs=126, mean epochs=106\n",
            "#14: score=1.138136, mean score=0.820913, stdev=0.221126,epochs=85, mean epochs=105\n",
            "#15: score=0.699105, mean score=0.812793, stdev=0.215778,epochs=108, mean epochs=105\n",
            "#16: score=0.900451, mean score=0.818271, stdev=0.210001,epochs=121, mean epochs=106\n",
            "#17: score=0.723284, mean score=0.812684, stdev=0.204953,epochs=118, mean epochs=107\n",
            "#18: score=1.070080, mean score=0.826984, stdev=0.207722,epochs=112, mean epochs=107\n",
            "#19: score=0.604149, mean score=0.815255, stdev=0.208215,epochs=135, mean epochs=108\n",
            "#20: score=0.651391, mean score=0.807062, stdev=0.206061,epochs=150, mean epochs=110\n",
            "#21: score=0.709781, mean score=0.802430, stdev=0.202159,epochs=116, mean epochs=111\n",
            "#22: score=0.603107, mean score=0.793370, stdev=0.201828,epochs=100, mean epochs=110\n",
            "#23: score=0.603426, mean score=0.785111, stdev=0.201156,epochs=122, mean epochs=111\n",
            "#24: score=0.690237, mean score=0.781158, stdev=0.197831,epochs=98, mean epochs=110\n",
            "#25: score=0.667345, mean score=0.776606, stdev=0.195113,epochs=132, mean epochs=111\n",
            "#26: score=0.767885, mean score=0.776270, stdev=0.191332,epochs=108, mean epochs=111\n",
            "#27: score=0.587335, mean score=0.769273, stdev=0.191115,epochs=102, mean epochs=110\n",
            "#28: score=0.663119, mean score=0.765481, stdev=0.188703,epochs=97, mean epochs=110\n",
            "#29: score=0.572984, mean score=0.758844, stdev=0.188718,epochs=98, mean epochs=110\n",
            "#30: score=0.540907, mean score=0.751579, stdev=0.189625,epochs=123, mean epochs=110\n",
            "#31: score=0.699984, mean score=0.749915, stdev=0.186765,epochs=115, mean epochs=110\n",
            "#32: score=0.546496, mean score=0.743558, stdev=0.187199,epochs=101, mean epochs=110\n",
            "#33: score=0.771817, mean score=0.744414, stdev=0.184405,epochs=135, mean epochs=111\n",
            "#34: score=1.154831, mean score=0.756485, stdev=0.194457,epochs=100, mean epochs=110\n",
            "#35: score=0.651458, mean score=0.753485, stdev=0.192456,epochs=147, mean epochs=111\n",
            "#36: score=0.561140, mean score=0.748142, stdev=0.192379,epochs=139, mean epochs=112\n",
            "#37: score=0.900896, mean score=0.752270, stdev=0.191371,epochs=134, mean epochs=113\n",
            "#38: score=0.723876, mean score=0.751523, stdev=0.188891,epochs=114, mean epochs=113\n",
            "#39: score=0.469784, mean score=0.744299, stdev=0.191698,epochs=118, mean epochs=113\n",
            "#40: score=0.633133, mean score=0.741520, stdev=0.190080,epochs=112, mean epochs=113\n",
            "#41: score=1.068246, mean score=0.749489, stdev=0.194395,epochs=132, mean epochs=113\n",
            "#42: score=0.547487, mean score=0.744679, stdev=0.194520,epochs=123, mean epochs=113\n",
            "#43: score=0.714780, mean score=0.743984, stdev=0.192298,epochs=122, mean epochs=114\n",
            "#44: score=0.679020, mean score=0.742507, stdev=0.190347,epochs=106, mean epochs=113\n",
            "#45: score=0.990288, mean score=0.748013, stdev=0.191731,epochs=106, mean epochs=113\n",
            "#46: score=0.594308, mean score=0.744672, stdev=0.190955,epochs=86, mean epochs=113\n",
            "#47: score=0.572750, mean score=0.741014, stdev=0.190535,epochs=124, mean epochs=113\n",
            "#48: score=0.817512, mean score=0.742608, stdev=0.188856,epochs=116, mean epochs=113\n",
            "#49: score=0.998190, mean score=0.747824, stdev=0.190380,epochs=125, mean epochs=113\n",
            "#50: score=0.620214, mean score=0.745272, stdev=0.189312,epochs=140, mean epochs=114\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KFXj09kIOb8Q"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}