01_neural_network_regression_with_tensorflow_video.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "01_neural_network_regression_with_tensorflow_video.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [
        "i6H4EkKAzxoL"
      ],
      "authorship_tag": "ABX9TyNdFjd9V1gIprT2dVL8wt+b",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kgene521/Calc/blob/master/01_neural_network_regression_with_tensorflow_video.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Neural Network Regression\n",
        "\n",
        "Housing prices dataset\n"
      ],
      "metadata": {
        "id": "i6H4EkKAzxoL"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dnczhkcezwdP"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from psutil import virtual_memory\n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "\n",
        "\n",
        "def print_env():\n",
        "  if ram_gb < 20:\n",
        "    print('Not using a high-RAM runtime')\n",
        "  else:\n",
        "    print('You are using a high-RAM runtime!')\n",
        "\n",
        "  gpu_info = !nvidia-smi\n",
        "  gpu_info = '\\n'.join(gpu_info)\n",
        "  if gpu_info.find('failed') >= 0:\n",
        "    print('Not connected to a GPU')\n",
        "  else:\n",
        "    print(gpu_info)\n",
        "\n",
        "def p(msg):\n",
        "  print(msg)\n",
        "\n",
        "def print_tensor_info(tensor, msg=\"\", conf=0):\n",
        "  nump_array = tensor.numpy()\n",
        "  print(\"=\"*50)\n",
        "  p(msg)\n",
        "  print(f\"{tensor}\")\n",
        "  if conf == 1:\n",
        "    print(f\"Number of dims (rank): {tensor.ndim}\")\n",
        "    print(f\"Shape of tensor: {tensor.shape}\")\n",
        "    print(f\"Datatype of every element: {tensor.dtype}\")\n",
        "    print(f\"Elements along the 0 axis: {tensor.shape[0]}\")\n",
        "    print(f\"Elements along the last axis: {tensor.shape[-1]}\")\n",
        "    print(f\"Total number of elements in tensor: {tf.size(tensor)}\", )\n",
        "    print(f\"Total number of elements in numpy array: {tf.size(tensor.numpy())}\", )\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Regression Model\n"
      ],
      "metadata": {
        "id": "UNLI5qB10ckD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "p(tf.__version__)"
      ],
      "metadata": {
        "id": "iL_w1Nkk0ySd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Create features\n",
        "X = np.array([-7., -4., -1., 2.0, 5.0, 8.0, 11.0, 14.0])\n",
        "\n",
        "# Create labels\n",
        "Y = np.array([3.0, 6.0, 9.0, 12.0, 15.0, 18.0, 21.0, 24.0])\n",
        "p(X), p(Y)\n",
        "plt.scatter(X, Y)\n",
        "y = X + 10\n",
        "p(y)\n",
        "y == X + 10"
      ],
      "metadata": {
        "id": "5QW0te6O3vEU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Input and Output Shapes"
      ],
      "metadata": {
        "id": "RzRxQBxc5kiv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Create features\n",
        "X = np.array([-7.0, -4.0, -1.0, 2.0, 5.0, 8.0, 11.0, 14.0])\n",
        "\n",
        "# Create labels\n",
        "y = np.array([3.0, 6.0, 9.0, 12.0, 15.0, 18.0, 21.0, 24.0])\n",
        "\n",
        "# Visualize it\n",
        "plt.scatter(X, y);\n",
        "\n",
        "p(X), p(y)\n",
        "\n"
      ],
      "metadata": {
        "id": "RD9rzPLg5DWC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Steps in Modeling in TensorFlow\n",
        "\n",
        "1. **Creating a model** - define the input and output layers and the hidden layers of a deep learning model\n",
        "2. **Compiling a model** - define the loss function, the optimizer, and the evaluation metrics (what we can use to interpret the performance of our model)\n",
        "3. **Fitting the model** - letting the model try to find patterns between X and Y (features and labels)\n",
        "4. **Evaluating the model** - see how well the model predicts"
      ],
      "metadata": {
        "id": "1BLWjAXS7pQ9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Create features (using tensors)\n",
        "X = tf.constant([-7.0, -4.0, -1.0, 2.0, 5.0, 8.0, 11.0, 14.0])\n",
        "\n",
        "# Create labels (using tensors)\n",
        "y = tf.constant([3.0, 6.0, 9.0, 12.0, 15.0, 18.0, 21.0, 24.0])\n",
        "\n",
        "# Visualize it\n",
        "p(X), p(y)\n",
        "plt.scatter(X, y);\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "rqoQrGqY7t0h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "\n",
        "# Create a model using the Keras Sequential API\n",
        "model = tf.keras.Sequential([\n",
        "  tf.keras.layers.Dense(1)\n",
        "])\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss=tf.keras.losses.mae,\n",
        "              optimizer=tf.keras.optimizers.SGD(),\n",
        "              metrics=[\"mae\"])\n",
        "\n",
        "# Fit the model\n",
        "# model.fit(X, y, epochs=5) # this breaks with Tensorflow 2.7.0+\n",
        "model.fit(tf.expand_dims(X, axis=-1), y, epochs=100)\n",
        "# Try and make a prediction using our model\n",
        "model.predict([17.0])\n",
        "\n"
      ],
      "metadata": {
        "id": "0lB3GMAyBWWl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Try and make a prediction using our model\n",
        "model.predict([17.0])"
      ],
      "metadata": {
        "id": "Tf8gJg2wOZL1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Improving the Model\n",
        "\n",
        "We can improve by altering the steps we took to create a model\n",
        "\n",
        "1. **Creating a model** - add more layers, increase the number of hidden units (all called neurons) within each of the hidden layers, change the activation function of each layer\n",
        "2. **Compiling a model** - change the optimization function, add/modify the **learning rate** of the optimization function\n",
        "3. **Fitting a model** - use more epochs, give the model more data\n"
      ],
      "metadata": {
        "id": "-UXCoPdeKCE3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "\n",
        "# Create a model using the Keras Sequential API\n",
        "model = tf.keras.Sequential([\n",
        "  tf.keras.layers.Dense(100, activation=\"relu\"),\n",
        "  tf.keras.layers.Dense(1)\n",
        "])\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss=tf.keras.losses.mae,\n",
        "              # optimizer=tf.keras.optimizers.Adam(lr=0.001),\n",
        "              optimizer=tf.keras.optimizers.SGD(),\n",
        "              metrics=[\"mae\"])\n",
        "\n",
        "# Fit the model\n",
        "# model.fit(X, y, epochs=5) # this breaks with Tensorflow 2.7.0+\n",
        "model.fit(tf.expand_dims(X, axis=-1), y, epochs=100)\n",
        "\n"
      ],
      "metadata": {
        "id": "AYJ8YkxzJeWO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.predict([17.0])\n"
      ],
      "metadata": {
        "id": "FlpMapDIjcig"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "\n",
        "# Create a model using the Keras Sequential API\n",
        "model = tf.keras.Sequential([\n",
        "  tf.keras.layers.Dense(50, activation=None),\n",
        "  tf.keras.layers.Dense(1)\n",
        "])\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss=tf.keras.losses.mae,\n",
        "              optimizer=tf.keras.optimizers.Adam(lr=0.01),\n",
        "              # optimizer=tf.keras.optimizers.SGD(),\n",
        "              metrics=[\"mae\"])\n",
        "\n",
        "# Fit the model\n",
        "# model.fit(X, y, epochs=5) # this breaks with Tensorflow 2.7.0+\n",
        "model.fit(tf.expand_dims(X, axis=-1), y, epochs=100)\n",
        "model.predict([17.0])[0][0]\n"
      ],
      "metadata": {
        "id": "cDNzZ7-Lkhmg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model.predict([17.0])[0][0]"
      ],
      "metadata": {
        "id": "z9GOrSIuknoB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "\n",
        "# Create a model using the Keras Sequential API\n",
        "model = tf.keras.Sequential([\n",
        "  tf.keras.layers.Dense(50, activation=None),\n",
        "  tf.keras.layers.Dense(1)\n",
        "])\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss=tf.keras.losses.mae,\n",
        "              optimizer=tf.keras.optimizers.Adam(lr=0.1),\n",
        "              # optimizer=tf.keras.optimizers.SGD(),\n",
        "              metrics=[\"mae\"])\n",
        "\n",
        "# Fit the model\n",
        "# model.fit(X, y, epochs=5) # this breaks with Tensorflow 2.7.0+\n",
        "model.fit(tf.expand_dims(X, axis=-1), y, epochs=100)\n",
        "model.predict([17.0])[0][0]\n"
      ],
      "metadata": {
        "id": "zBO3aUZ5nU3K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "p(model.predict([17.0])), p(model.predict([17.0])[0][0])"
      ],
      "metadata": {
        "id": "5J1CsWS_nqeq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Common ways to improve a model\n",
        "1. Adding/Removing layers\n",
        "2. Increase/Decrease the number of neurons in hidden layers\n",
        "3. Change/Remove the activation function\n",
        "4. Change the optimization function\n",
        "5. Change the learning rate (up/down)\n",
        "6. Fitting on a larger dataset\n",
        "7. Fitting for more epochs"
      ],
      "metadata": {
        "id": "GDX33OtRoP3w"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Evaluating a model\n",
        "\n",
        "Build a model --> fit it --> evaluate it --> tweak the model --> etc.\n",
        "\n",
        "> Visualize, Visualize, Visualize\n"
      ],
      "metadata": {
        "id": "QIP1Fd3totZR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "X = tf.range(-100, 100, 4)\n",
        "y = X + 10\n",
        "X, y"
      ],
      "metadata": {
        "id": "wdFu3sxbpxdJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tf.random.set_seed(42)\n",
        "\n",
        "# Create a model using the Keras Sequential API\n",
        "model = tf.keras.Sequential([\n",
        "  tf.keras.layers.Dense(50, activation=None),\n",
        "  tf.keras.layers.Dense(1)\n",
        "])\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss=tf.keras.losses.mae,\n",
        "              optimizer=tf.keras.optimizers.Adam(lr=0.1),\n",
        "              # optimizer=tf.keras.optimizers.SGD(),\n",
        "              metrics=[\"mae\"])\n",
        "\n",
        "# Fit the model\n",
        "# model.fit(X, y, epochs=5) # this breaks with Tensorflow 2.7.0+\n",
        "model.fit(tf.expand_dims(X, axis=-1), y, epochs=100)\n",
        "p(\"=\"*80)\n",
        "print(model.predict([17.0])[0][0])\n"
      ],
      "metadata": {
        "id": "2-BTaF8QrZ06"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.plot(X, y), plt.scatter(X, y)"
      ],
      "metadata": {
        "id": "vo4kzmiDs3yp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Pandas versison: train_ds, val_ds, test_ds\n",
        "def get_dataset_partitions_pd(df, \n",
        "                              train_split=0.8, \n",
        "                              val_split=0.1, \n",
        "                              test_split=0.1):\n",
        "    assert (train_split + test_split + val_split) == 1\n",
        "    \n",
        "    # Only allows for equal validation and test splits\n",
        "    assert val_split == test_split \n",
        "    df_size = len(df)\n",
        "\n",
        "    # Specify seed to always have the same split distribution between runs\n",
        "    df_sample = df.sample(frac=1, random_state=12)\n",
        "    indices_or_sections = [int(train_split * df_size), \n",
        "                           int((1 - val_split - test_split) * df_size)]\n",
        "    \n",
        "    train_ds, val_ds, test_ds = np.split(df_sample, indices_or_sections)\n",
        "    \n",
        "    return train_ds, val_ds, test_ds\n",
        "\n",
        "# TF version: train_ds, val_ds, test_ds\n",
        "def get_dataset_partitions_tf(ds, \n",
        "                              train_split=0.8, \n",
        "                              val_split=0.1, \n",
        "                              test_split=0.1):\n",
        "    ds_size = X.shape[0]\n",
        "\n",
        "    assert (train_split + test_split + val_split) == 1\n",
        "    \n",
        "    # Specify seed to always have the same split distribution between runs\n",
        "    ds = tf.random.shuffle(ds, seed=42)\n",
        "\n",
        "    train_size = int(train_split * ds_size)\n",
        "    val_size = int(val_split * ds_size)\n",
        "    \n",
        "    train_ds = ds.take(train_size)    \n",
        "    val_ds = ds.skip(train_size).take(val_size)\n",
        "    test_ds = ds.skip(train_size).skip(val_size)\n",
        "    \n",
        "    return train_ds, val_ds, test_ds\n",
        "\n",
        "a, b, c = get_dataset_partitions_tf(X)"
      ],
      "metadata": {
        "id": "Lytz2oPL3gHa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Training Set**\n",
        "\n",
        "**Validation Set**\n",
        "\n",
        "**Test Set**\n"
      ],
      "metadata": {
        "id": "hWMjLAHB7x5X"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import logging\n",
        "logger = logging.getLogger()\n",
        "logger.setLevel(logging.CRITICAL)\n",
        "\n",
        "# Use TF splitting version\n",
        "tf.random.set_seed(42)\n",
        "\n",
        "X_train, X_val, X_test = get_dataset_partitions_tf(ds=X, train_split=0.8, val_split=0.1, test_split=0.1)\n",
        "\n",
        "# Create a model using the Keras Sequential API\n",
        "model = tf.keras.Sequential([\n",
        "  tf.keras.layers.Dense(50, activation=None),\n",
        "  tf.keras.layers.Dense(1)\n",
        "])\n",
        "\n",
        "# Compile the model\n",
        "model.compile(loss=tf.keras.losses.mae,\n",
        "              optimizer=tf.keras.optimizers.Adam(lr=0.1),\n",
        "              # optimizer=tf.keras.optimizers.SGD(),\n",
        "              metrics=[\"mae\"])\n",
        "\n",
        "# Fit the model\n",
        "# model.fit(X, y, epochs=5) # this breaks with Tensorflow 2.7.0+\n",
        "model.fit(tf.expand_dims(X_train, axis=-1), y, epochs=100)\n",
        "p(\"=\"*80)\n",
        "print(model.predict([17.0])[0][0])\n",
        "\n"
      ],
      "metadata": {
        "id": "Vqm6Mxqt4Nwo"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}