ImprovingNN1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ImprovingNN1.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/pkraison/ex.ai/blob/master/ImprovingNN1.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "c3Gn3NLmYyZn",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Gradient Checking\n",
        "You are part of a team working to make mobile transactions available globally, and are asked to build a deep learning model to detect fraud--whenever someone makes a transaction, you want to see if the transaction might be fraudulent, such as if the user's account has been taken over by a hacker.\n",
        "\n",
        "But backpropagation is quite challenging to implement, and sometimes has bugs. Because this is a mission-critical application, your company's CEO wants to be really certain that your implementation of backpropagation is correct. Your CEO says, \"Give me a proof that your backpropagation is actually working!\" To give this reassurance, you are going to use \"gradient checking\""
      ]
    },
    {
      "metadata": {
        "id": "lXfHx7gvYY6j",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Packages\n",
        "import numpy as np\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "leMhU3w-ZRfG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def gradient_check_n_test_case(): \n",
        "    np.random.seed(1)\n",
        "    x = np.random.randn(4,3)\n",
        "    y = np.array([1, 1, 0])\n",
        "    W1 = np.random.randn(5,4) \n",
        "    b1 = np.random.randn(5,1) \n",
        "    W2 = np.random.randn(3,5) \n",
        "    b2 = np.random.randn(3,1) \n",
        "    W3 = np.random.randn(1,3) \n",
        "    b3 = np.random.randn(1,1) \n",
        "    parameters = {\"W1\": W1,\n",
        "                  \"b1\": b1,\n",
        "                  \"W2\": W2,\n",
        "                  \"b2\": b2,\n",
        "                  \"W3\": W3,\n",
        "                  \"b3\": b3}\n",
        "\n",
        "    \n",
        "    return x, y, parameters"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zDUsKduOh9Fm",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def sigmoid(x):\n",
        "    \"\"\"\n",
        "    Compute the sigmoid of x\n",
        "\n",
        "    Arguments:\n",
        "    x -- A scalar or numpy array of any size.\n",
        "\n",
        "    Return:\n",
        "    s -- sigmoid(x)\n",
        "    \"\"\"\n",
        "    s = 1/(1+np.exp(-x))\n",
        "    return s\n",
        "\n",
        "def relu(x):\n",
        "    \"\"\"\n",
        "    Compute the relu of x\n",
        "\n",
        "    Arguments:\n",
        "    x -- A scalar or numpy array of any size.\n",
        "\n",
        "    Return:\n",
        "    s -- relu(x)\n",
        "    \"\"\"\n",
        "    s = np.maximum(0,x)\n",
        "    \n",
        "    return s"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FWicyYhpiFVf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def dictionary_to_vector(parameters):\n",
        "    \"\"\"\n",
        "    Roll all our parameters dictionary into a single vector satisfying our specific required shape.\n",
        "    \"\"\"\n",
        "    keys = []\n",
        "    count = 0\n",
        "    for key in [\"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\"]:\n",
        "        \n",
        "        # flatten parameter\n",
        "        new_vector = np.reshape(parameters[key], (-1,1))\n",
        "        keys = keys + [key]*new_vector.shape[0]\n",
        "        \n",
        "        if count == 0:\n",
        "            theta = new_vector\n",
        "        else:\n",
        "            theta = np.concatenate((theta, new_vector), axis=0)\n",
        "        count = count + 1\n",
        "\n",
        "    return theta, keys"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "yZWgYMkUiLon",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def vector_to_dictionary(theta):\n",
        "    \"\"\"\n",
        "    Unroll all our parameters dictionary from a single vector satisfying our specific required shape.\n",
        "    \"\"\"\n",
        "    parameters = {}\n",
        "    parameters[\"W1\"] = theta[:20].reshape((5,4))\n",
        "    parameters[\"b1\"] = theta[20:25].reshape((5,1))\n",
        "    parameters[\"W2\"] = theta[25:40].reshape((3,5))\n",
        "    parameters[\"b2\"] = theta[40:43].reshape((3,1))\n",
        "    parameters[\"W3\"] = theta[43:46].reshape((1,3))\n",
        "    parameters[\"b3\"] = theta[46:47].reshape((1,1))\n",
        "\n",
        "    return parameters"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "M0VI9UBUhqMs",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def gradients_to_vector(gradients):\n",
        "    \"\"\"\n",
        "    Roll all our gradients dictionary into a single vector satisfying our specific required shape.\n",
        "    \"\"\"\n",
        "    \n",
        "    count = 0\n",
        "    for key in [\"dW1\", \"db1\", \"dW2\", \"db2\", \"dW3\", \"db3\"]:\n",
        "        # flatten parameter\n",
        "        new_vector = np.reshape(gradients[key], (-1,1))\n",
        "        \n",
        "        if count == 0:\n",
        "            theta = new_vector\n",
        "        else:\n",
        "            theta = np.concatenate((theta, new_vector), axis=0)\n",
        "        count = count + 1\n",
        "\n",
        "    return theta"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "YR-dYNsxibCO",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "## 1) How does gradient checking work?\n",
        "\n",
        "Backpropagation computes the gradients $\\frac{\\partial J}{\\partial \\theta}$, where $\\theta$ denotes the parameters of the model. $J$ is computed using forward propagation and your loss function.\n",
        "\n",
        "Because forward propagation is relatively easy to implement, you're confident you got that right, and so you're almost  100% sure that you're computing the cost $J$ correctly. Thus, you can use your code for computing $J$ to verify the code for computing $\\frac{\\partial J}{\\partial \\theta}$. \n",
        "\n",
        "Let's look back at the definition of a derivative (or gradient):\n",
        "$$ \\frac{\\partial J}{\\partial \\theta} = \\lim_{\\varepsilon \\to 0} \\frac{J(\\theta + \\varepsilon) - J(\\theta - \\varepsilon)}{2 \\varepsilon} \\tag{1}$$\n",
        "\n",
        "If you're not familiar with the \"$\\displaystyle \\lim_{\\varepsilon \\to 0}$\" notation, it's just a way of saying \"when $\\varepsilon$ is really really small.\"\n",
        "\n",
        "We know the following:\n",
        "\n",
        "- $\\frac{\\partial J}{\\partial \\theta}$ is what you want to make sure you're computing correctly. \n",
        "- You can compute $J(\\theta + \\varepsilon)$ and $J(\\theta - \\varepsilon)$ (in the case that $\\theta$ is a real number), since you're confident your implementation for $J$ is correct. \n",
        "\n",
        "Lets use equation (1) and a small value for $\\varepsilon$ to convince your CEO that your code for computing  $\\frac{\\partial J}{\\partial \\theta}$ is correct!"
      ]
    },
    {
      "metadata": {
        "id": "3hEqeROuh1ux",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def forward_propagation(x, theta):\n",
        "    \"\"\"\n",
        "    Implement the linear forward propagation (compute J) presented in Figure 1 (J(theta) = theta * x)\n",
        "    \n",
        "    Arguments:\n",
        "    x -- a real-valued input\n",
        "    theta -- our parameter, a real number as well\n",
        "    \n",
        "    Returns:\n",
        "    J -- the value of function J, computed using the formula J(theta) = theta * x\n",
        "    \"\"\"\n",
        "    J = theta * x\n",
        "    \n",
        "    return J"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6htGlfH3jbCk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5cf9f0e5-dfda-42f4-f1ea-4cf763922581"
      },
      "cell_type": "code",
      "source": [
        "x, theta = 2, 4\n",
        "J = forward_propagation(x, theta)\n",
        "print (\"J = \" + str(J))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "J = 8\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "wCX9aR2Wjeyz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def backward_propagation(x, theta):\n",
        "    \"\"\"\n",
        "    Computes the derivative of J with respect to theta (see Figure 1).\n",
        "    \n",
        "    Arguments:\n",
        "    x -- a real-valued input\n",
        "    theta -- our parameter, a real number as well\n",
        "    \n",
        "    Returns:\n",
        "    dtheta -- the gradient of the cost with respect to theta\n",
        "    \"\"\"\n",
        "    \n",
        "    \n",
        "    dtheta = x\n",
        "    \n",
        "    return dtheta"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "QLH96iqkjorp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6cee66a3-a77e-4fb7-ec14-fb7305f78360"
      },
      "cell_type": "code",
      "source": [
        "x, theta = 2, 4\n",
        "dtheta = backward_propagation(x, theta)\n",
        "print (\"dtheta = \" + str(dtheta))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dtheta = 2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "hW5nbWvEjsMR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def gradient_check(x, theta, epsilon = 1e-7):\n",
        "    \"\"\"\n",
        "    Implement the backward propagation presented in Figure 1.\n",
        "    \n",
        "    Arguments:\n",
        "    x -- a real-valued input\n",
        "    theta -- our parameter, a real number as well\n",
        "    epsilon -- tiny shift to the input to compute approximated gradient with formula(1)\n",
        "    \n",
        "    Returns:\n",
        "    difference -- difference (2) between the approximated gradient and the backward propagation gradient\n",
        "    \"\"\"\n",
        "    \n",
        "    # Compute gradapprox using left side of formula (1). epsilon is small enough, you don't need to worry about the limit.\n",
        "    \n",
        "    thetaplus = theta+ epsilon                              # Step 1\n",
        "    thetaminus = theta-epsilon                              # Step 2\n",
        "    J_plus = forward_propagation(x, thetaplus)                                 # Step 3\n",
        "    J_minus = forward_propagation(x, thetaminus)                                 # Step 4\n",
        "    gradapprox = (J_plus-J_minus)/2/epsilon                              # Step 5\n",
        "    \n",
        "    \n",
        "    # Check if gradapprox is close enough to the output of backward_propagation()\n",
        "    \n",
        "    grad = backward_propagation(x, theta)\n",
        "    \n",
        "    \n",
        "    \n",
        "    numerator = np.linalg.norm(grad-gradapprox)                               # Step 1'\n",
        "    denominator = np.linalg.norm(grad)+np.linalg.norm(gradapprox)                             # Step 2'\n",
        "    difference = numerator/denominator                              # Step 3'\n",
        "    \n",
        "    \n",
        "    if difference < 1e-7:\n",
        "        print (\"The gradient is correct!\")\n",
        "    else:\n",
        "        print (\"The gradient is wrong!\")\n",
        "    \n",
        "    return difference"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "qpOEwi99kFj2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "2d6de2c4-6a22-4c83-a255-e14a27a441b6"
      },
      "cell_type": "code",
      "source": [
        "x, theta = 2, 4\n",
        "difference = gradient_check(x, theta)\n",
        "print(\"difference = \" + str(difference))"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The gradient is correct!\n",
            "difference = 2.919335883291695e-10\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ca-VISuFkNw5",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def forward_propagation_n(X, Y, parameters):\n",
        "    \"\"\"\n",
        "    Implements the forward propagation (and computes the cost) presented in Figure 3.\n",
        "    \n",
        "    Arguments:\n",
        "    X -- training set for m examples\n",
        "    Y -- labels for m examples \n",
        "    parameters -- python dictionary containing your parameters \"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\":\n",
        "                    W1 -- weight matrix of shape (5, 4)\n",
        "                    b1 -- bias vector of shape (5, 1)\n",
        "                    W2 -- weight matrix of shape (3, 5)\n",
        "                    b2 -- bias vector of shape (3, 1)\n",
        "                    W3 -- weight matrix of shape (1, 3)\n",
        "                    b3 -- bias vector of shape (1, 1)\n",
        "    \n",
        "    Returns:\n",
        "    cost -- the cost function (logistic cost for one example)\n",
        "    \"\"\"\n",
        "    \n",
        "    # retrieve parameters\n",
        "    m = X.shape[1]\n",
        "    W1 = parameters[\"W1\"]\n",
        "    b1 = parameters[\"b1\"]\n",
        "    W2 = parameters[\"W2\"]\n",
        "    b2 = parameters[\"b2\"]\n",
        "    W3 = parameters[\"W3\"]\n",
        "    b3 = parameters[\"b3\"]\n",
        "\n",
        "    # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID\n",
        "    Z1 = np.dot(W1, X) + b1\n",
        "    A1 = relu(Z1)\n",
        "    Z2 = np.dot(W2, A1) + b2\n",
        "    A2 = relu(Z2)\n",
        "    Z3 = np.dot(W3, A2) + b3\n",
        "    A3 = sigmoid(Z3)\n",
        "\n",
        "    # Cost\n",
        "    logprobs = np.multiply(-np.log(A3),Y) + np.multiply(-np.log(1 - A3), 1 - Y)\n",
        "    cost = 1./m * np.sum(logprobs)\n",
        "    \n",
        "    cache = (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3)\n",
        "    \n",
        "    return cost, cache"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "gL2-i92JkUQ9",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def backward_propagation_n(X, Y, cache):\n",
        "    \"\"\"\n",
        "    Implement the backward propagation presented in figure 2.\n",
        "    \n",
        "    Arguments:\n",
        "    X -- input datapoint, of shape (input size, 1)\n",
        "    Y -- true \"label\"\n",
        "    cache -- cache output from forward_propagation_n()\n",
        "    \n",
        "    Returns:\n",
        "    gradients -- A dictionary with the gradients of the cost with respect to each parameter, activation and pre-activation variables.\n",
        "    \"\"\"\n",
        "    \n",
        "    m = X.shape[1]\n",
        "    (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) = cache\n",
        "    \n",
        "    dZ3 = A3 - Y\n",
        "    dW3 = 1./m * np.dot(dZ3, A2.T)\n",
        "    db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True)\n",
        "    \n",
        "    dA2 = np.dot(W3.T, dZ3)\n",
        "    dZ2 = np.multiply(dA2, np.int64(A2 > 0))\n",
        "    dW2 = 1./m * np.dot(dZ2, A1.T) \n",
        "    db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True)\n",
        "    \n",
        "    dA1 = np.dot(W2.T, dZ2)\n",
        "    dZ1 = np.multiply(dA1, np.int64(A1 > 0))\n",
        "    dW1 = 1./m * np.dot(dZ1, X.T)\n",
        "    db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True)\n",
        "    \n",
        "    gradients = {\"dZ3\": dZ3, \"dW3\": dW3, \"db3\": db3,\n",
        "                 \"dA2\": dA2, \"dZ2\": dZ2, \"dW2\": dW2, \"db2\": db2,\n",
        "                 \"dA1\": dA1, \"dZ1\": dZ1, \"dW1\": dW1, \"db1\": db1}\n",
        "    \n",
        "    return gradients"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "5oIg4Nq1knFR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def gradient_check_n(parameters, gradients, X, Y, epsilon = 1e-7):\n",
        "    \"\"\"\n",
        "    Checks if backward_propagation_n computes correctly the gradient of the cost output by forward_propagation_n\n",
        "    \n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters \"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\":\n",
        "    grad -- output of backward_propagation_n, contains gradients of the cost with respect to the parameters. \n",
        "    x -- input datapoint, of shape (input size, 1)\n",
        "    y -- true \"label\"\n",
        "    epsilon -- tiny shift to the input to compute approximated gradient with formula(1)\n",
        "    \n",
        "    Returns:\n",
        "    difference -- difference (2) between the approximated gradient and the backward propagation gradient\n",
        "    \"\"\"\n",
        "    \n",
        "    # Set-up variables\n",
        "    parameters_values, _ = dictionary_to_vector(parameters)\n",
        "    grad = gradients_to_vector(gradients)\n",
        "    num_parameters = parameters_values.shape[0]\n",
        "    J_plus = np.zeros((num_parameters, 1))\n",
        "    J_minus = np.zeros((num_parameters, 1))\n",
        "    gradapprox = np.zeros((num_parameters, 1))\n",
        "    \n",
        "    # Compute gradapprox\n",
        "    for i in range(num_parameters):\n",
        "        \n",
        "        # Compute J_plus[i]. Inputs: \"parameters_values, epsilon\". Output = \"J_plus[i]\".\n",
        "        # \"_\" is used because the function you have to outputs two parameters but we only care about the first one\n",
        "        \n",
        "        thetaplus =  np.copy(parameters_values)                                      # Step 1\n",
        "        thetaplus[i][0] = thetaplus[i][0]+epsilon                                # Step 2\n",
        "        J_plus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary(thetaplus))                                  # Step 3\n",
        "       \n",
        "        \n",
        "        # Compute J_minus[i]. Inputs: \"parameters_values, epsilon\". Output = \"J_minus[i]\".\n",
        "        \n",
        "        thetaminus = np.copy(parameters_values)                                     # Step 1\n",
        "        thetaminus[i][0] = thetaplus[i][0]-epsilon                                # Step 2        \n",
        "        J_minus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary(thetaminus))                                   # Step 3\n",
        "        \n",
        "        \n",
        "        # Compute gradapprox[i]\n",
        "        \n",
        "        gradapprox[i] = (J_plus[i]-J_minus[i])/(2*epsilon)\n",
        "        \n",
        "    \n",
        "    # Compare gradapprox to backward propagation gradients by computing difference.\n",
        "    \n",
        "    numerator = np.linalg.norm(gradapprox-grad)                                           # Step 1'\n",
        "    denominator = np.linalg.norm(grad)+np.linalg.norm(gradapprox)                                         # Step 2'\n",
        "    difference = numerator/denominator                                          # Step 3'\n",
        "    \n",
        "\n",
        "    if difference > 1e-7:\n",
        "        print (\"\\033[93m\" + \"There is a mistake in the backward propagation! difference = \" + str(difference) + \"\\033[0m\")\n",
        "    else:\n",
        "        print (\"\\033[92m\" + \"Your backward propagation works perfectly fine! difference = \" + str(difference) + \"\\033[0m\")\n",
        "    \n",
        "    return difference"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "oMhZlUYMk6-2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9e07f58c-b0ad-4a26-cca1-4dc1516a1531"
      },
      "cell_type": "code",
      "source": [
        "X, Y, parameters = gradient_check_n_test_case()\n",
        "\n",
        "cost, cache = forward_propagation_n(X, Y, parameters)\n",
        "gradients = backward_propagation_n(X, Y, cache)\n",
        "difference = gradient_check_n(parameters, gradients, X, Y)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[93mThere is a mistake in the backward propagation! difference = 0.3333333478985919\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}