C2_W1_Gradient_Checking.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNZFwLspHBo1EEY4Z6w8WeF"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "def sigmoid(x):\n",
        "    \"\"\"\n",
        "    Compute the sigmoid of x\n",
        "\n",
        "    Arguments:\n",
        "    x -- A scalar or numpy array of any size.\n",
        "\n",
        "    Return: \n",
        "    s -- sigmoid(x)\n",
        "    \"\"\"\n",
        "    s = 1/(1+np.exp(-x))\n",
        "    return s\n",
        "\n",
        "def relu(x):\n",
        "    \"\"\"\n",
        "    Compute the relu of x\n",
        "\n",
        "    Arguments:\n",
        "    x -- A scalar or numpy array of any size.\n",
        "\n",
        "    Return:\n",
        "    s -- relu(x)\n",
        "    \"\"\"\n",
        "    s = np.maximum(0,x)\n",
        "    \n",
        "    return s\n",
        "\n",
        "def dictionary_to_vector(parameters):\n",
        "    \"\"\"\n",
        "    Roll all our parameters dictionary into a single vector satisfying our specific required shape.\n",
        "    \"\"\"\n",
        "    keys = []\n",
        "    count = 0\n",
        "    for key in [\"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\"]:\n",
        "        \n",
        "        # flatten parameter\n",
        "        new_vector = np.reshape(parameters[key], (-1,1))\n",
        "        keys = keys + [key]*new_vector.shape[0]\n",
        "        \n",
        "        if count == 0:\n",
        "            theta = new_vector\n",
        "        else:\n",
        "            theta = np.concatenate((theta, new_vector), axis=0)\n",
        "        count = count + 1\n",
        "\n",
        "    return theta, keys\n",
        "\n",
        "def vector_to_dictionary(theta):\n",
        "    \"\"\"\n",
        "    Unroll all our parameters dictionary from a single vector satisfying our specific required shape.\n",
        "    \"\"\"\n",
        "    parameters = {}\n",
        "    parameters[\"W1\"] = theta[:20].reshape((5,4))\n",
        "    parameters[\"b1\"] = theta[20:25].reshape((5,1))\n",
        "    parameters[\"W2\"] = theta[25:40].reshape((3,5))\n",
        "    parameters[\"b2\"] = theta[40:43].reshape((3,1))\n",
        "    parameters[\"W3\"] = theta[43:46].reshape((1,3))\n",
        "    parameters[\"b3\"] = theta[46:47].reshape((1,1))\n",
        "\n",
        "    return parameters\n",
        "\n",
        "def gradients_to_vector(gradients):\n",
        "    \"\"\"\n",
        "    Roll all our gradients dictionary into a single vector satisfying our specific required shape.\n",
        "    \"\"\"\n",
        "    \n",
        "    count = 0\n",
        "    for key in [\"dW1\", \"db1\", \"dW2\", \"db2\", \"dW3\", \"db3\"]:\n",
        "        # flatten parameter\n",
        "        new_vector = np.reshape(gradients[key], (-1,1))\n",
        "        \n",
        "        if count == 0:\n",
        "            theta = new_vector\n",
        "        else:\n",
        "            theta = np.concatenate((theta, new_vector), axis=0)\n",
        "        count = count + 1\n",
        "\n",
        "    return theta"
      ],
      "metadata": {
        "id": "e8J5BorezCE0"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# GRADED FUNCTION: forward_propagation\n",
        "\n",
        "def forward_propagation(x, theta):\n",
        "    \"\"\"\n",
        "    Implement the linear forward propagation (compute J) presented in Figure 1 (J(theta) = theta * x)\n",
        "    \n",
        "    Arguments:\n",
        "    x -- a real-valued input\n",
        "    theta -- our parameter, a real number as well\n",
        "    \n",
        "    Returns:\n",
        "    J -- the value of function J, computed using the formula J(theta) = theta * x\n",
        "    \"\"\"\n",
        "    \n",
        "    # (approx. 1 line)\n",
        "    # J = \n",
        "    # YOUR CODE STARTS HERE\n",
        "    J = np.dot(x,theta)\n",
        "    # YOUR CODE ENDS HERE\n",
        "    return J"
      ],
      "metadata": {
        "id": "Npkj6jbszU3l"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x, theta = 2, 4\n",
        "J = forward_propagation(x, theta)\n",
        "print (\"J = \" + str(J))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zj3NPn2l2qz6",
        "outputId": "81259074-fc46-4dcd-d0f5-cb7f6f7f524e"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "J = 8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# GRADED FUNCTION: backward_propagation\n",
        "\n",
        "def backward_propagation(x, theta):\n",
        "    \"\"\"\n",
        "    Computes the derivative of J with respect to theta (see Figure 1).\n",
        "    \n",
        "    Arguments:\n",
        "    x -- a real-valued input\n",
        "    theta -- our parameter, a real number as well\n",
        "    \n",
        "    Returns:\n",
        "    dtheta -- the gradient of the cost with respect to theta\n",
        "    \"\"\"\n",
        "    \n",
        "    # (approx. 1 line)\n",
        "    # dtheta = \n",
        "    # YOUR CODE STARTS HERE\n",
        "    dtheta = x\n",
        "    # YOUR CODE ENDS HERE\n",
        "    return dtheta"
      ],
      "metadata": {
        "id": "VItBbaq723b6"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x, theta = 3, 4\n",
        "dtheta = backward_propagation(x, theta)\n",
        "print (\"dtheta = \" + str(dtheta))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fb52M9Fj3DE4",
        "outputId": "90828a64-fa6a-4a4e-e7d9-a628c667dced"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "dtheta = 3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def gradient_check(x, theta, epsilon = 1e-7):\n",
        "    \"\"\"\n",
        "    Implement the backward propagation presented in Figure 1.\n",
        "    \n",
        "    Arguments:\n",
        "    x -- a real-valued input\n",
        "    theta -- our parameter, a real number as well\n",
        "    epsilon -- tiny shift to the input to compute approximated gradient with formula(1)\n",
        "    \n",
        "    Returns:\n",
        "    difference -- difference (2) between the approximated gradient and the backward propagation gradient\n",
        "    \"\"\"\n",
        "    \n",
        "    # Compute gradapprox using left side of formula (1). epsilon is small enough, you don't need to worry about the limit.\n",
        "    ### START CODE HERE ### (approx. 5 lines)\n",
        "    thetaplus = theta + epsilon                           # Step 1\n",
        "    thetaminus = theta - epsilon                          # Step 2\n",
        "    J_plus = np.dot(thetaplus,x)                            # Step 3\n",
        "    J_minus = np.dot(thetaminus,x)                        # Step 4\n",
        "    gradapprox = (J_plus - J_minus)/(2*epsilon)                              # Step 5\n",
        "    ### END CODE HERE ###\n",
        "    \n",
        "    # Check if gradapprox is close enough to the output of backward_propagation()\n",
        "    ### START CODE HERE ### (approx. 1 line)\n",
        "    grad = x\n",
        "    ### END CODE HERE ###\n",
        "    \n",
        "    ### START CODE HERE ### (approx. 1 line)\n",
        "    numerator = np.linalg.norm(gradapprox-grad)                         # Step 1'\n",
        "    denominator = np.linalg.norm(gradapprox) + np.linalg.norm(grad)     # Step 2'\n",
        "    difference = numerator/denominator                                  # Step 3'\n",
        "    ### END CODE HERE ###\n",
        "    \n",
        "    if difference < 1e-7:\n",
        "        print (\"The gradient is correct! \", difference)\n",
        "    else:\n",
        "        print (\"The gradient is wrong! \", difference)\n",
        "    \n",
        "    return difference"
      ],
      "metadata": {
        "id": "hA5zGk5Y3fBo"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x, theta = 2, 4\n",
        "difference = gradient_check(x, theta)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BjKxm1iG5JLU",
        "outputId": "b049d1bb-c577-4d1e-a45e-3dcaadbbfa4d"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "The gradient is correct!  2.919335883291695e-10\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def forward_propagation_n(X, Y, parameters):\n",
        "    \"\"\"\n",
        "    Implements the forward propagation (and computes the cost) presented in Figure 3.\n",
        "    \n",
        "    Arguments:\n",
        "    X -- training set for m examples\n",
        "    Y -- labels for m examples \n",
        "    parameters -- python dictionary containing your parameters \"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\":\n",
        "                    W1 -- weight matrix of shape (5, 4)\n",
        "                    b1 -- bias vector of shape (5, 1)\n",
        "                    W2 -- weight matrix of shape (3, 5)\n",
        "                    b2 -- bias vector of shape (3, 1)\n",
        "                    W3 -- weight matrix of shape (1, 3)\n",
        "                    b3 -- bias vector of shape (1, 1)\n",
        "    \n",
        "    Returns:\n",
        "    cost -- the cost function (logistic cost for one example)\n",
        "    cache -- a tuple with the intermediate values (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3)\n",
        "\n",
        "    \"\"\"\n",
        "    \n",
        "    # retrieve parameters\n",
        "    m = X.shape[1]\n",
        "    W1 = parameters[\"W1\"]\n",
        "    b1 = parameters[\"b1\"]\n",
        "    W2 = parameters[\"W2\"]\n",
        "    b2 = parameters[\"b2\"]\n",
        "    W3 = parameters[\"W3\"]\n",
        "    b3 = parameters[\"b3\"]\n",
        "\n",
        "    # LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID\n",
        "    Z1 = np.dot(W1, X) + b1\n",
        "    A1 = relu(Z1)\n",
        "    Z2 = np.dot(W2, A1) + b2\n",
        "    A2 = relu(Z2)\n",
        "    Z3 = np.dot(W3, A2) + b3\n",
        "    A3 = sigmoid(Z3)\n",
        "\n",
        "    # Cost\n",
        "    log_probs = np.multiply(-np.log(A3),Y) + np.multiply(-np.log(1 - A3), 1 - Y)\n",
        "    cost = 1. / m * np.sum(log_probs)\n",
        "    \n",
        "    cache = (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3)\n",
        "    \n",
        "    return cost, cache"
      ],
      "metadata": {
        "id": "c0ouaGVc-u8t"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def backward_propagation_n(X, Y, cache):\n",
        "    \"\"\"\n",
        "    Implement the backward propagation presented in figure 2.\n",
        "    \n",
        "    Arguments:\n",
        "    X -- input datapoint, of shape (input size, 1)\n",
        "    Y -- true \"label\"\n",
        "    cache -- cache output from forward_propagation_n()\n",
        "    \n",
        "    Returns:\n",
        "    gradients -- A dictionary with the gradients of the cost with respect to each parameter, activation and pre-activation variables.\n",
        "    \"\"\"\n",
        "    \n",
        "    m = X.shape[1]\n",
        "    (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) = cache\n",
        "    \n",
        "    dZ3 = A3 - Y\n",
        "    dW3 = 1. / m * np.dot(dZ3, A2.T)\n",
        "    db3 = 1. / m * np.sum(dZ3, axis=1, keepdims=True)\n",
        "    \n",
        "    dA2 = np.dot(W3.T, dZ3)\n",
        "    dZ2 = np.multiply(dA2, np.int64(A2 > 0))\n",
        "    dW2 = 1. / m * np.dot(dZ2, A1.T) * 2\n",
        "    db2 = 1. / m * np.sum(dZ2, axis=1, keepdims=True)\n",
        "    \n",
        "    dA1 = np.dot(W2.T, dZ2)\n",
        "    dZ1 = np.multiply(dA1, np.int64(A1 > 0))\n",
        "    dW1 = 1. / m * np.dot(dZ1, X.T)\n",
        "    db1 = 4. / m * np.sum(dZ1, axis=1, keepdims=True)\n",
        "    \n",
        "    gradients = {\"dZ3\": dZ3, \"dW3\": dW3, \"db3\": db3,\n",
        "                 \"dA2\": dA2, \"dZ2\": dZ2, \"dW2\": dW2, \"db2\": db2,\n",
        "                 \"dA1\": dA1, \"dZ1\": dZ1, \"dW1\": dW1, \"db1\": db1}\n",
        "    \n",
        "    return gradients"
      ],
      "metadata": {
        "id": "vsep8gRM_1Yg"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# GRADED FUNCTION: gradient_check_n\n",
        "\n",
        "def gradient_check_n(parameters, gradients, X, Y, epsilon = 1e-7):\n",
        "    \"\"\"\n",
        "    Checks if backward_propagation_n computes correctly the gradient of the cost output by forward_propagation_n\n",
        "    \n",
        "    Arguments:\n",
        "    parameters -- python dictionary containing your parameters \"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\":\n",
        "    grad -- output of backward_propagation_n, contains gradients of the cost with respect to the parameters. \n",
        "    x -- input datapoint, of shape (input size, 1)\n",
        "    y -- true \"label\"\n",
        "    epsilon -- tiny shift to the input to compute approximated gradient with formula(1)\n",
        "    \n",
        "    Returns:\n",
        "    difference -- difference (2) between the approximated gradient and the backward propagation gradient\n",
        "    \"\"\"\n",
        "    \n",
        "    # Set-up variables\n",
        "    parameters_values, _ = dictionary_to_vector(parameters)\n",
        "    grad = gradients_to_vector(gradients)\n",
        "    num_parameters = parameters_values.shape[0]\n",
        "    J_plus = np.zeros((num_parameters, 1))\n",
        "    J_minus = np.zeros((num_parameters, 1))\n",
        "    gradapprox = np.zeros((num_parameters, 1))\n",
        "    \n",
        "    # Compute gradapprox\n",
        "    for i in range(num_parameters):\n",
        "        \n",
        "        # Compute J_plus[i]. Inputs: \"parameters_values, epsilon\". Output = \"J_plus[i]\".\n",
        "        # \"_\" is used because the function you have to outputs two parameters but we only care about the first one\n",
        "        ### START CODE HERE ### (approx. 3 lines)\n",
        "        thetaplus = np.copy(parameters_values)                                       # Step 1\n",
        "        thetaplus[i][0] = thetaplus[i][0] + epsilon                                  # Step 2\n",
        "        J_plus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary(thetaplus))  # Step 3\n",
        "        ### END CODE HERE ###\n",
        "        \n",
        "        # Compute J_minus[i]. Inputs: \"parameters_values, epsilon\". Output = \"J_minus[i]\".\n",
        "        ### START CODE HERE ### (approx. 3 lines)\n",
        "        thetaminus = np.copy(parameters_values)                                       # Step 1\n",
        "        thetaminus[i][0] = thetaminus[i][0] - epsilon                                 # Step 2\n",
        "        J_minus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary(thetaminus)) # Step 3\n",
        "        ### END CODE HERE ###\n",
        "        \n",
        "        # Compute gradapprox[i]\n",
        "        ### START CODE HERE ### (approx. 1 line)\n",
        "        gradapprox[i] = (J_plus[i] - J_minus[i])/(2*epsilon)\n",
        "        ### END CODE HERE ###\n",
        "    \n",
        "    # Compare gradapprox to backward propagation gradients by computing difference.\n",
        "    ### START CODE HERE ### (approx. 1 line)\n",
        "    numerator = np.linalg.norm(grad - gradapprox)                       # Step 1'\n",
        "    denominator = np.linalg.norm(grad) + np.linalg.norm(gradapprox)     # Step 2'\n",
        "    difference = numerator/denominator                                  # Step 3'\n",
        "    ### END CODE HERE ###\n",
        "\n",
        "    if difference > 2e-7:\n",
        "        print (\"\\033[93m\" + \"There is a mistake in the backward propagation! difference = \" + str(difference) + \"\\033[0m\")\n",
        "    else:\n",
        "        print (\"\\033[92m\" + \"Your backward propagation works perfectly fine! difference = \" + str(difference) + \"\\033[0m\")\n",
        "    \n",
        "    return difference"
      ],
      "metadata": {
        "id": "rRa-_GQ0A1Dt"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def gradient_check_n_test_case(): \n",
        "  np.random.seed(1)\n",
        "  x = np.random.randn(4,3)\n",
        "  y = np.array([1, 1, 0])\n",
        "  W1 = np.random.randn(5,4) \n",
        "  b1 = np.random.randn(5,1) \n",
        "  W2 = np.random.randn(3,5) \n",
        "  b2 = np.random.randn(3,1) \n",
        "  W3 = np.random.randn(1,3) \n",
        "  b3 = np.random.randn(1,1) \n",
        "  parameters = {\"W1\": W1,\n",
        "                \"b1\": b1,\n",
        "                \"W2\": W2,\n",
        "                \"b2\": b2,\n",
        "                \"W3\": W3,\n",
        "                \"b3\": b3}  \n",
        "  return x, y,parameters"
      ],
      "metadata": {
        "id": "ctzsu5kiH5D8"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "X, Y, parameters = gradient_check_n_test_case()\n",
        "cost, cache = forward_propagation_n(X, Y, parameters)\n",
        "gradients = backward_propagation_n(X, Y, cache)\n",
        "difference = gradient_check_n(parameters, gradients, X, Y, 1e-7)\n",
        "expected_values = [0.2850931567761623, 1.1890913024229996e-07]\n",
        "assert not(type(difference) == np.ndarray), \"You are not using np.linalg.norm for numerator or denominator\"\n",
        "assert np.any(np.isclose(difference, expected_values)), \"Wrong value. It is not one of the expected values\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qEEeHNW8BBdx",
        "outputId": "482080ce-0beb-450e-b92d-a5c6ae959501"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[93mThere is a mistake in the backward propagation! difference = 0.2850931567761624\u001b[0m\n"
          ]
        }
      ]
    }
  ]
}