009.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 - Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.5/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n",
      "/opt/conda/lib/python3.5/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import h5py\n",
    "import matplotlib.pyplot as plt\n",
    "from testCases_v4 import *\n",
    "from dnn_utils_v2 import sigmoid, sigmoid_backward, relu, relu_backward\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['figure.figsize'] = (5.0, 4.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'\n",
    "plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "# dnn_utils provides some necessary functions for this notebook.\n",
    "# testCases provides some test cases to assess the correctness of your functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 2 - Helper Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- The model's structure is *[LINEAR -> RELU] $ \\times$ (L-1) -> LINEAR -> SIGMOID*. I.e., it has $L-1$ layers using a ReLU activation function followed by an output layer with a sigmoid activation function.\n",
    "- Use random initialization for the weight matrices. Use `np.random.randn(shape) * 0.01`.\n",
    "- Use zeros initialization for the biases. Use `np.zeros(shape)`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def initialize_parameters_deep(layer_dims):\n",
    "    \"\"\"\n",
    "    Arguments:\n",
    "    layer_dims -- python array (list) containing the dimensions of each layer in our network\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your parameters \"W1\", \"b1\", ..., \"WL\", \"bL\":\n",
    "                    Wl -- weight matrix of shape (layer_dims[l], layer_dims[l-1])\n",
    "                    bl -- bias vector of shape (layer_dims[l], 1)\n",
    "    \"\"\"\n",
    "    \n",
    "    np.random.seed(3)\n",
    "    parameters = {}\n",
    "    L = len(layer_dims)            # number of layers in the network\n",
    "\n",
    "    for l in range(1, L):\n",
    "        parameters['W' + str(l)] = np.random.randn(layer_dims[l], layer_dims[l-1]) * 0.01\n",
    "        parameters['b' + str(l)] = np.zeros((layer_dims[l], 1))\n",
    "        \n",
    "        assert(parameters['W' + str(l)].shape == (layer_dims[l], layer_dims[l-1]))\n",
    "        assert(parameters['b' + str(l)].shape == (layer_dims[l], 1))\n",
    "\n",
    "        \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 - Forward propagation module\n",
    "\n",
    "### 3.1 - Linear Forward \n",
    "\n",
    "You will complete three functions in this order:\n",
    "\n",
    "- LINEAR\n",
    "- LINEAR -> ACTIVATION where ACTIVATION will be either ReLU or Sigmoid. \n",
    "- [LINEAR -> RELU] $\\times$ (L-1) -> LINEAR -> SIGMOID (whole model)\n",
    "\n",
    "The linear forward module (vectorized over all the examples):\n",
    "\n",
    "$$Z^{[l]} = W^{[l]}A^{[l-1]} +b^{[l]}\\tag{4}$$\n",
    "\n",
    "where $A^{[0]} = X$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_forward(A, W, b):\n",
    "    \"\"\"\n",
    "    Implement the linear part of a layer's forward propagation.\n",
    "\n",
    "    Arguments:\n",
    "    A -- activations from previous layer (or input data): (size of previous layer, number of examples)\n",
    "    W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)\n",
    "    b -- bias vector, numpy array of shape (size of the current layer, 1)\n",
    "\n",
    "    Returns:\n",
    "    Z -- the input of the activation function, also called pre-activation parameter \n",
    "    cache -- a python dictionary containing \"A\", \"W\" and \"b\" ; stored for computing the backward pass efficiently\n",
    "    \"\"\"\n",
    "    \n",
    "    Z = W @ A + b\n",
    "    \n",
    "    assert(Z.shape == (W.shape[0], A.shape[1]))\n",
    "    cache = (A, W, b)\n",
    "    \n",
    "    return Z, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def linear_activation_forward(A_prev, W, b, activation):\n",
    "    \"\"\"\n",
    "    Implement the forward propagation for the LINEAR->ACTIVATION layer\n",
    "\n",
    "    Arguments:\n",
    "    A_prev -- activations from previous layer (or input data): (size of previous layer, number of examples)\n",
    "    W -- weights matrix: numpy array of shape (size of current layer, size of previous layer)\n",
    "    b -- bias vector, numpy array of shape (size of the current layer, 1)\n",
    "    activation -- the activation to be used in this layer, stored as a text string: \"sigmoid\" or \"relu\"\n",
    "\n",
    "    Returns:\n",
    "    A -- the output of the activation function, also called the post-activation value \n",
    "    cache -- a python dictionary containing \"linear_cache\" and \"activation_cache\";\n",
    "             stored for computing the backward pass efficiently\n",
    "    \"\"\"\n",
    "    \n",
    "    if activation == \"sigmoid\":\n",
    "        # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\".\n",
    "        Z, linear_cache = linear_forward(A_prev, W, b)\n",
    "        A, activation_cache = sigmoid(Z)\n",
    "    \n",
    "    elif activation == \"relu\":\n",
    "        # Inputs: \"A_prev, W, b\". Outputs: \"A, activation_cache\".\n",
    "        Z, linear_cache = linear_forward(A_prev, W, b)\n",
    "        A, activation_cache = relu(Z)\n",
    "    \n",
    "    assert (A.shape == (W.shape[0], A_prev.shape[1]))\n",
    "    cache = (linear_cache, activation_cache)\n",
    "\n",
    "    return A, cache"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def L_model_forward(X, parameters):\n",
    "    \"\"\"\n",
    "    Implement forward propagation for the [LINEAR->RELU]*(L-1)->LINEAR->SIGMOID computation\n",
    "    \n",
    "    Arguments:\n",
    "    X -- data, numpy array of shape (input size, number of examples)\n",
    "    parameters -- output of initialize_parameters_deep()\n",
    "    \n",
    "    Returns:\n",
    "    AL -- last post-activation value\n",
    "    caches -- list of caches containing:\n",
    "                every cache of linear_activation_forward() (there are L-1 of them, indexed from 0 to L-1)\n",
    "    \"\"\"\n",
    "\n",
    "    caches = []\n",
    "    A = X\n",
    "    L = len(parameters) // 2                  # number of layers in the neural network\n",
    "    \n",
    "    # Implement [LINEAR -> RELU]*(L-1). Add \"cache\" to the \"caches\" list.\n",
    "    for l in range(1, L):\n",
    "        A_prev = A \n",
    "        A, cache = linear_activation_forward(A_prev, parameters['W{:d}'.format(l)], parameters['b{:d}'.format(l)], activation='relu')\n",
    "        caches.append(cache)\n",
    "    \n",
    "    # Implement LINEAR -> SIGMOID. Add \"cache\" to the \"caches\" list.\n",
    "    AL, cache = linear_activation_forward(A, parameters['W%d' % L], parameters['b%d' % L], activation='sigmoid')\n",
    "    caches.append(cache)\n",
    "    \n",
    "    assert(AL.shape == (1,X.shape[1]))\n",
    "            \n",
    "    return AL, caches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 - Cost function\n",
    "\n",
    "$$-\\frac{1}{m} \\sum\\limits_{i = 1}^{m} (y^{(i)}\\log\\left(a^{[L] (i)}\\right) + (1-y^{(i)})\\log\\left(1- a^{[L](i)}\\right)) \\tag{7}$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GRADED FUNCTION: compute_cost\n",
    "\n",
    "def compute_cost(AL, Y):\n",
    "    \"\"\"\n",
    "    Implement the cost function defined by equation (7).\n",
    "\n",
    "    Arguments:\n",
    "    AL -- probability vector corresponding to your label predictions, shape (1, number of examples)\n",
    "    Y -- true \"label\" vector (for example: containing 0 if non-cat, 1 if cat), shape (1, number of examples)\n",
    "\n",
    "    Returns:\n",
    "    cost -- cross-entropy cost\n",
    "    \"\"\"\n",
    "    \n",
    "    m = Y.shape[1]\n",
    "\n",
    "    # Compute loss from aL and y.\n",
    "    cost = -1 / m * np.sum(Y * np.log(AL) + (1-Y) * np.log(1-AL))\n",
    "    \n",
    "    cost = np.squeeze(cost)      # To make sure your cost's shape is what we expect (e.g. this turns [[17]] into 17).\n",
    "    assert(cost.shape == ())\n",
    "    \n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 - Backward propagation module\n",
    "\n",
    "- LINEAR backward\n",
    "- LINEAR -> ACTIVATION backward where ACTIVATION computes the derivative of either the ReLU or sigmoid activation\n",
    "- [LINEAR -> RELU] $\\times$ (L-1) -> LINEAR -> SIGMOID backward (whole model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 - Linear backward\n",
    "\n",
    "Suppose you have already calculated the derivative $dZ^{[l]} = \\frac{\\partial \\mathcal{L} }{\\partial Z^{[l]}}$. You want to get $(dW^{[l]}, db^{[l]} dA^{[l-1]})$.\n",
    "\n",
    "The three outputs $(dW^{[l]}, db^{[l]}, dA^{[l]})$ are computed using the input $dZ^{[l]}$.Here are the formulas you need:\n",
    "$$ dW^{[l]} = \\frac{\\partial \\mathcal{L} }{\\partial W^{[l]}} = \\frac{1}{m} dZ^{[l]} A^{[l-1] T} \\tag{8}$$\n",
    "$$ db^{[l]} = \\frac{\\partial \\mathcal{L} }{\\partial b^{[l]}} = \\frac{1}{m} \\sum_{i = 1}^{m} dZ^{[l](i)}\\tag{9}$$\n",
    "$$ dA^{[l-1]} = \\frac{\\partial \\mathcal{L} }{\\partial A^{[l-1]}} = W^{[l] T} dZ^{[l]} \\tag{10}$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear_backward(dZ, cache):\n",
    "    \"\"\"\n",
    "    Implement the linear portion of backward propagation for a single layer (layer l)\n",
    "\n",
    "    Arguments:\n",
    "    dZ -- Gradient of the cost with respect to the linear output (of current layer l)\n",
    "    cache -- tuple of values (A_prev, W, b) coming from the forward propagation in the current layer\n",
    "\n",
    "    Returns:\n",
    "    dA_prev -- Gradient of the cost with respect to the activation (of the previous layer l-1), same shape as A_prev\n",
    "    dW -- Gradient of the cost with respect to W (current layer l), same shape as W\n",
    "    db -- Gradient of the cost with respect to b (current layer l), same shape as b\n",
    "    \"\"\"\n",
    "    A_prev, W, b = cache\n",
    "    m = A_prev.shape[1]\n",
    "    \n",
    "    dW = 1 / m * dZ @ A_prev.T\n",
    "    db = 1 / m * np.sum(dZ, axis=1, keepdims=True)\n",
    "    dA_prev = W.T @ dZ\n",
    "    \n",
    "    assert (dA_prev.shape == A_prev.shape)\n",
    "    assert (dW.shape == W.shape)\n",
    "    assert (db.shape == b.shape)\n",
    "    \n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 - Linear-Activation backward\n",
    "\n",
    "Next, you will create a function that merges the two helper functions: **`linear_backward`** and the backward step for the activation **`linear_activation_backward`**. \n",
    "\n",
    "To help you implement `linear_activation_backward`, we provided two backward functions:\n",
    "- **`sigmoid_backward`**: Implements the backward propagation for SIGMOID unit. You can call it as follows:\n",
    "\n",
    "```python\n",
    "dZ = sigmoid_backward(dA, activation_cache)\n",
    "```\n",
    "\n",
    "- **`relu_backward`**: Implements the backward propagation for RELU unit. You can call it as follows:\n",
    "\n",
    "```python\n",
    "dZ = relu_backward(dA, activation_cache)\n",
    "```\n",
    "\n",
    "If $g(.)$ is the activation function, \n",
    "`sigmoid_backward` and `relu_backward` compute $$dZ^{[l]} = dA^{[l]} * g'(Z^{[l]}) \\tag{11}$$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# GRADED FUNCTION: linear_activation_backward\n",
    "\n",
    "def linear_activation_backward(dA, cache, activation):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for the LINEAR->ACTIVATION layer.\n",
    "    \n",
    "    Arguments:\n",
    "    dA -- post-activation gradient for current layer l \n",
    "    cache -- tuple of values (linear_cache, activation_cache) we store for computing backward propagation efficiently\n",
    "    activation -- the activation to be used in this layer, stored as a text string: \"sigmoid\" or \"relu\"\n",
    "    \n",
    "    Returns:\n",
    "    dA_prev -- Gradient of the cost with respect to the activation (of the previous layer l-1), same shape as A_prev\n",
    "    dW -- Gradient of the cost with respect to W (current layer l), same shape as W\n",
    "    db -- Gradient of the cost with respect to b (current layer l), same shape as b\n",
    "    \"\"\"\n",
    "    linear_cache, activation_cache = cache\n",
    "    \n",
    "    if activation == \"relu\":\n",
    "        ### START CODE HERE ### (≈ 2 lines of code)\n",
    "        dZ = relu_backward(dA, activation_cache)\n",
    "        dA_prev, dW, db = linear_backward(dZ, linear_cache)\n",
    "        ### END CODE HERE ###\n",
    "        \n",
    "    elif activation == \"sigmoid\":\n",
    "        ### START CODE HERE ### (≈ 2 lines of code)\n",
    "        dZ = sigmoid_backward(dA, activation_cache)\n",
    "        dA_prev, dW, db = linear_backward(dZ, linear_cache)\n",
    "        ### END CODE HERE ###\n",
    "    \n",
    "    return dA_prev, dW, db"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3 - L-Model Backward \n",
    "\n",
    "** Initializing backpropagation**:\n",
    "To backpropagate through this network, we know that the output is, \n",
    "$A^{[L]} = \\sigma(Z^{[L]})$. Your code thus needs to compute `dAL` $= \\frac{\\partial \\mathcal{L}}{\\partial A^{[L]}}$:\n",
    "```python\n",
    "dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL)) # derivative of cost with respect to AL\n",
    "```\n",
    "\n",
    "You can then use this post-activation gradient `dAL` to keep going backward: \n",
    "\n",
    "$$grads[\"dW\" + str(l)] = dW^{[l]}\\tag{15} $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def L_model_backward(AL, Y, caches):\n",
    "    \"\"\"\n",
    "    Implement the backward propagation for the [LINEAR->RELU] * (L-1) -> LINEAR -> SIGMOID group\n",
    "    \n",
    "    Arguments:\n",
    "    AL -- probability vector, output of the forward propagation (L_model_forward())\n",
    "    Y -- true \"label\" vector (containing 0 if non-cat, 1 if cat)\n",
    "    caches -- list of caches containing:\n",
    "                every cache of linear_activation_forward() with \"relu\" (it's caches[l], for l in range(L-1) i.e l = 0...L-2)\n",
    "                the cache of linear_activation_forward() with \"sigmoid\" (it's caches[L-1])\n",
    "    \n",
    "    Returns:\n",
    "    grads -- A dictionary with the gradients\n",
    "             grads[\"dA\" + str(l)] = ... \n",
    "             grads[\"dW\" + str(l)] = ...\n",
    "             grads[\"db\" + str(l)] = ... \n",
    "    \"\"\"\n",
    "    grads = {}\n",
    "    L = len(caches) # the number of layers\n",
    "    m = AL.shape[1]\n",
    "    Y = Y.reshape(AL.shape) # after this line, Y is the same shape as AL\n",
    "    \n",
    "    # Initializing the backpropagation\n",
    "    dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))\n",
    "    \n",
    "    # Lth layer (SIGMOID -> LINEAR) gradients. Inputs: \"dAL, current_cache\". Outputs: \"grads[\"dAL-1\"], grads[\"dWL\"], grads[\"dbL\"]\n",
    "    current_cache = caches[L-1]\n",
    "    grads[\"dA\" + str(L-1)], grads[\"dW\" + str(L)], grads[\"db\" + str(L)] = linear_activation_backward(dAL, current_cache, 'sigmoid')\n",
    "    \n",
    "    # Loop from l=L-2 to l=0\n",
    "    for l in reversed(range(L-1)):\n",
    "        # lth layer: (RELU -> LINEAR) gradients.\n",
    "        # Inputs: \"grads[\"dA\" + str(l + 1)], current_cache\". Outputs: \"grads[\"dA\" + str(l)] , grads[\"dW\" + str(l + 1)] , grads[\"db\" + str(l + 1)] \n",
    "        current_cache = caches[l]\n",
    "        dA_prev_temp, dW_temp, db_temp = linear_activation_backward(grads[\"dA\" + str(l + 1)], current_cache, 'relu')\n",
    "        grads[\"dA\" + str(l)] = dA_prev_temp\n",
    "        grads[\"dW\" + str(l + 1)] = dW_temp\n",
    "        grads[\"db\" + str(l + 1)] = db_temp\n",
    "\n",
    "    return grads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def update_parameters(parameters, grads, learning_rate):\n",
    "    \"\"\"\n",
    "    Update parameters using gradient descent\n",
    "    \n",
    "    Arguments:\n",
    "    parameters -- python dictionary containing your parameters \n",
    "    grads -- python dictionary containing your gradients, output of L_model_backward\n",
    "    \n",
    "    Returns:\n",
    "    parameters -- python dictionary containing your updated parameters \n",
    "                  parameters[\"W\" + str(l)] = ... \n",
    "                  parameters[\"b\" + str(l)] = ...\n",
    "    \"\"\"\n",
    "    \n",
    "    L = len(parameters) // 2 # number of layers in the neural network\n",
    "\n",
    "    # Update rule for each parameter. Use a for loop.\n",
    "    for l in range(L):\n",
    "        parameters[\"W\" + str(l+1)] = parameters[\"W\" + str(l+1)] - learning_rate * grads[\"dW\" + str(l+1)]\n",
    "        parameters[\"b\" + str(l+1)] = parameters[\"b\" + str(l+1)] - learning_rate * grads[\"db\" + str(l+1)]\n",
    "    return parameters"
   ]
  }
 ],
 "metadata": {
  "coursera": {
   "course_slug": "neural-networks-deep-learning",
   "graded_item_id": "c4HO0",
   "launcher_item_id": "lSYZM"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}