layers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import MaxPooling2D,Dense,Flatten,Conv2D,Dense,GlobalMaxPooling1D \n",
    "from tensorflow.keras.models import Sequential, Model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## نگاهی عمیق تر به لایه ها\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### دسترسی به وزن های لایه ها\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/mobilenet/mobilenet_1_0_224_tf.h5\n",
      "17227776/17225924 [==============================] - 3s 0us/step\n"
     ]
    }
   ],
   "source": [
    "model = tf.keras.applications.MobileNet(weights= 'imagenet',\n",
    "                                         include_top = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"mobilenet_1.00_224\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_8 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
      "_________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)    (None, 225, 225, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv1 (Conv2D)               (None, 112, 112, 32)      864       \n",
      "_________________________________________________________________\n",
      "conv1_bn (BatchNormalization (None, 112, 112, 32)      128       \n",
      "_________________________________________________________________\n",
      "conv1_relu (ReLU)            (None, 112, 112, 32)      0         \n",
      "_________________________________________________________________\n",
      "conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 32)      288       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_bn (BatchNormaliza (None, 112, 112, 32)      128       \n",
      "_________________________________________________________________\n",
      "conv_dw_1_relu (ReLU)        (None, 112, 112, 32)      0         \n",
      "_________________________________________________________________\n",
      "conv_pw_1 (Conv2D)           (None, 112, 112, 64)      2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_1_bn (BatchNormaliza (None, 112, 112, 64)      256       \n",
      "_________________________________________________________________\n",
      "conv_pw_1_relu (ReLU)        (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "conv_pad_2 (ZeroPadding2D)   (None, 113, 113, 64)      0         \n",
      "_________________________________________________________________\n",
      "conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 64)        576       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_bn (BatchNormaliza (None, 56, 56, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_relu (ReLU)        (None, 56, 56, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_2 (Conv2D)           (None, 56, 56, 128)       8192      \n",
      "_________________________________________________________________\n",
      "conv_pw_2_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_2_relu (ReLU)        (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_3_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_3_relu (ReLU)        (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_3 (Conv2D)           (None, 56, 56, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_3_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_3_relu (ReLU)        (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pad_4 (ZeroPadding2D)   (None, 57, 57, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_4_bn (BatchNormaliza (None, 28, 28, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_4_relu (ReLU)        (None, 28, 28, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_4 (Conv2D)           (None, 28, 28, 256)       32768     \n",
      "_________________________________________________________________\n",
      "conv_pw_4_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_4_relu (ReLU)        (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 256)       2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_5_relu (ReLU)        (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_5 (Conv2D)           (None, 28, 28, 256)       65536     \n",
      "_________________________________________________________________\n",
      "conv_pw_5_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_5_relu (ReLU)        (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_pad_6 (ZeroPadding2D)   (None, 29, 29, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 256)       2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_6_bn (BatchNormaliza (None, 14, 14, 256)       1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_6_relu (ReLU)        (None, 14, 14, 256)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_6 (Conv2D)           (None, 14, 14, 512)       131072    \n",
      "_________________________________________________________________\n",
      "conv_pw_6_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_6_relu (ReLU)        (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_7_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_7_relu (ReLU)        (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_7 (Conv2D)           (None, 14, 14, 512)       262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_7_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_7_relu (ReLU)        (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_8_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_8_relu (ReLU)        (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_8 (Conv2D)           (None, 14, 14, 512)       262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_8_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_8_relu (ReLU)        (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_9_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_9_relu (ReLU)        (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_9 (Conv2D)           (None, 14, 14, 512)       262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_9_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_9_relu (ReLU)        (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_10 (DepthwiseConv2D) (None, 14, 14, 512)       4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_10_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_10_relu (ReLU)       (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_10 (Conv2D)          (None, 14, 14, 512)       262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_10_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_10_relu (ReLU)       (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_11 (DepthwiseConv2D) (None, 14, 14, 512)       4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_11_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_11_relu (ReLU)       (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_11 (Conv2D)          (None, 14, 14, 512)       262144    \n",
      "_________________________________________________________________\n",
      "conv_pw_11_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_11_relu (ReLU)       (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 512)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_12 (DepthwiseConv2D) (None, 7, 7, 512)         4608      \n",
      "_________________________________________________________________\n",
      "conv_dw_12_bn (BatchNormaliz (None, 7, 7, 512)         2048      \n",
      "_________________________________________________________________\n",
      "conv_dw_12_relu (ReLU)       (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_12 (Conv2D)          (None, 7, 7, 1024)        524288    \n",
      "_________________________________________________________________\n",
      "conv_pw_12_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "conv_pw_12_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_13 (DepthwiseConv2D) (None, 7, 7, 1024)        9216      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_13 (Conv2D)          (None, 7, 7, 1024)        1048576   \n",
      "_________________________________________________________________\n",
      "conv_pw_13_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
      "_________________________________________________________________\n",
      "conv_pw_13_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d (Gl (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "reshape_1 (Reshape)          (None, 1, 1, 1024)        0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 1, 1, 1024)        0         \n",
      "_________________________________________________________________\n",
      "conv_preds (Conv2D)          (None, 1, 1, 1000)        1025000   \n",
      "_________________________________________________________________\n",
      "reshape_2 (Reshape)          (None, 1000)              0         \n",
      "_________________________________________________________________\n",
      "predictions (Activation)     (None, 1000)              0         \n",
      "=================================================================\n",
      "Total params: 4,253,864\n",
      "Trainable params: 4,231,976\n",
      "Non-trainable params: 21,888\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "اطلاعات مربوط به لایه های یک شبکه در مشخصه `layers` آن ذخیره شده است."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "93"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model.layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.convolutional.Conv2D at 0x17023165dc8>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " برای این که به وزن های یک لایه دسترسی پیدا کنیم (اگر آن لایه وزنی داشته باشد) با مشخصه `weights` قابل دسترسی هستند."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'conv1_bn_2/gamma:0' shape=(32,) dtype=float32, numpy=\n",
       "array([ 0.97061753,  0.6502549 ,  0.2254817 ,  0.52415645,  0.98184997,\n",
       "        0.6788391 ,  1.1129196 ,  0.34201714,  0.6771581 ,  3.3115928 ,\n",
       "        0.83776003,  1.0192714 ,  0.30443215,  0.4222344 , -0.2554651 ,\n",
       "        0.96727175,  0.90101403,  0.86320865,  0.86195886,  0.7156465 ,\n",
       "        1.0060233 ,  0.53907305,  1.3887119 ,  0.24158667,  0.7830971 ,\n",
       "        0.672174  ,  3.0135353 ,  1.195009  ,  1.8365439 ,  0.47269213,\n",
       "        0.45962656,  1.0022334 ], dtype=float32)>"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[3].weights[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.97061753,  0.6502549 ,  0.2254817 ,  0.52415645,  0.98184997,\n",
       "        0.6788391 ,  1.1129196 ,  0.34201714,  0.6771581 ,  3.3115928 ,\n",
       "        0.83776003,  1.0192714 ,  0.30443215,  0.4222344 , -0.2554651 ,\n",
       "        0.96727175,  0.90101403,  0.86320865,  0.86195886,  0.7156465 ,\n",
       "        1.0060233 ,  0.53907305,  1.3887119 ,  0.24158667,  0.7830971 ,\n",
       "        0.672174  ,  3.0135353 ,  1.195009  ,  1.8365439 ,  0.47269213,\n",
       "        0.45962656,  1.0022334 ], dtype=float32)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model.layers[3].get_weights()[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "bias های یک لایه هم با مشخصه `bias` قابل دستترسی هستند."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(10,), dtype=float32, numpy=\n",
       "array([-0.48195136,  0.02448254, -0.98349696, -1.0550407 ,  0.1832502 ,\n",
       "       -0.12676108,  0.20039281, -0.91389215, -1.1940612 , -1.5195633 ],\n",
       "      dtype=float32)>"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[90].bias[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 1024, 1000), dtype=float32, numpy=\n",
       "array([[[-0.00136319, -0.05997207, -0.00945091, ..., -0.12088422,\n",
       "          0.04772888,  0.10453417],\n",
       "        [-0.09380977,  0.0133763 , -0.00086663, ..., -0.05051607,\n",
       "         -0.05330694, -0.00499932],\n",
       "        [ 0.03785546,  0.02643952,  0.03207906, ...,  0.06625971,\n",
       "         -0.10837644, -0.02584988],\n",
       "        ...,\n",
       "        [-0.01420777, -0.08234182, -0.05374264, ...,  0.11939939,\n",
       "         -0.04741941, -0.02354525],\n",
       "        [-0.02142392, -0.0442558 ,  0.03908358, ..., -0.0139262 ,\n",
       "          0.14381737, -0.11744042],\n",
       "        [-0.06758978,  0.06164968,  0.1083982 , ...,  0.06461229,\n",
       "         -0.01201531,  0.05659112]]], dtype=float32)>"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[90].kernel[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "برای دسترسی به یک لایه یک راه دیگر استفاده از نام آن لایه به جای ایندکس است که به خصوص وقتی شبکه ما بزرگ است به کار می آید.\n",
    " مثلا فرض کنید می خواهیم به آخرین لایه کانولوشنال شبکه دسترسی پیدا کنیم."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.convolutional.Conv2D at 0x1700833fb48>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model.get_layer('conv5_block3_3_conv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "وزن ها به صورت مشابه قابل دسترسی هستند.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'conv5_block3_3_conv_3/kernel:0' shape=(1, 1, 512, 2048) dtype=float32, numpy=\n",
       " array([[[[-0.00772033,  0.0285685 , -0.0311284 , ..., -0.00376943,\n",
       "            0.00396393, -0.01859178],\n",
       "          [ 0.03072583, -0.02049609,  0.00851315, ..., -0.03072169,\n",
       "            0.09537231, -0.01491823],\n",
       "          [ 0.00430369,  0.00874794, -0.00027265, ..., -0.00887997,\n",
       "            0.00560437, -0.02863607],\n",
       "          ...,\n",
       "          [ 0.01828984, -0.01196646, -0.00106144, ..., -0.00687517,\n",
       "            0.00607187, -0.04117712],\n",
       "          [-0.00993001, -0.00776328, -0.02632025, ..., -0.02515809,\n",
       "            0.01466563,  0.00475405],\n",
       "          [ 0.00103569,  0.01750279,  0.00685418, ...,  0.02908638,\n",
       "           -0.00971192,  0.02699419]]]], dtype=float32)>,\n",
       " <tf.Variable 'conv5_block3_3_conv_3/bias:0' shape=(2048,) dtype=float32, numpy=\n",
       " array([-1.5847744e-08, -5.5954280e-07,  9.1473552e-08, ...,\n",
       "        -3.5979106e-08, -9.8053647e-07, -1.8873895e-07], dtype=float32)>]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model.get_layer('conv5_block3_3_conv').weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model.get_layer('conv5_block1_2_pad').weights "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "#func_model = Model( inputs = base_model, \n",
    "#outputs = Dense(10,activation = 'softmax'))\n",
    "#func_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### دسترسی پیدا کردن به ورودی و خروجی لایه ها"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'conv5_block1_1_relu_3/Identity:0' shape=(None, 7, 7, 512) dtype=float32>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model.get_layer('conv5_block1_2_pad').input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'conv5_block1_2_pad_3/Identity:0' shape=(None, 9, 9, 512) dtype=float32>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model.get_layer('conv5_block1_2_pad').output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'input_4:0' shape=(None, 224, 224, 3) dtype=float32>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "base_model.input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense (Dense)                (None, 224, 224, 10)      40        \n",
      "=================================================================\n",
      "Total params: 40\n",
      "Trainable params: 40\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "seq_model = Sequential([base_model.input, \n",
    "Dense(10,activation = 'softmax')])\n",
    "seq_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### فریز کردن وزن های لایه ها"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "پیش از این در خصوص فریز کردن وزن های مدل برای انتقال یادگیری به صورت مفصل بحث کردیم. در این جا می خواهیم ببینیم که چطور می توانیم وزن های یک لایه خاص  از مدل را فریز کنیم."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:`input_shape` is undefined or non-square, or `rows` is not in [128, 160, 192, 224]. Weights for input shape (224, 224) will be loaded as the default.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'input_15:0' shape=(None, 150, 150, 3) dtype=float32>"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = tf.keras.applications.MobileNet(weights= 'imagenet',\n",
    "                                        input_shape=(150,150,3),\n",
    "                                         include_top = False)\n",
    "#model.summary()\n",
    "model.input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "وقتی که یک لایه را به مدل اضافه می کنیم می توانیم با قرار دادن `trainable = False` کاری کنیم که وزن های آن لایه فریز شوند و در زمان آموزش به روز رسانی نشوند.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'input_12:0' shape=(None, None, None, 3) dtype=float32>"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_1 = Sequential([\n",
    "    model,\n",
    "    Flatten(),\n",
    "    Dense(100,activation = 'relu', trainable = False),\n",
    "    Dense(10,activation = 'softmax')\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "با استفاده از API فاکشنال به صورت زیر خواهد بود"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "inputs = model.input #keras.layers.Input(shape = (150,150,3))\n",
    "x = model(inputs,training = False)\n",
    "x = Flatten()(x)\n",
    "x = Dense(128,activation='relu',trainable = False)(x)\n",
    "outputs = Dense(5,activation='softmax' )(x)\n",
    "final_model = Model(inputs,outputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "mobilenet_1.00_224 (Model)   (None, None, None, 1024)  3228864   \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, None)              0         \n",
      "=================================================================\n",
      "Total params: 3,228,864\n",
      "Trainable params: 2,158,400\n",
      "Non-trainable params: 1,070,464\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_1.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "یک راه دیگر هم استفاده از متد `get_layer()` استفاده است."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.get_layer('conv_pw_13').trainable = False\n",
    "model.get_layer('conv_pw_13_relu').trainable = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "باید مدل را کامپایل کنیم تا  این تغییرات اعمال شوند."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss = 'categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "دقت کنید که این که می توانیم لایه های مختلف شبکه را فریز کنیم به این معنا نیست که این کار کار خوبی است و صرفا به عنوان مثال این کار را انجام دهیم."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### لایه های lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ساخت لایه های اختصاصی"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "برای ساخت یک لایه کراس سه تابع را باید در زمان ایجاد کلاس این لایه پیاده سازی کنیم:\n",
    "\n",
    "1. یک تابع که وزن های قابل آموزش مدل را تعریف می کند. (تابع build)\n",
    "\n",
    "2. یک تابع که منطق لایه را مشخص می کند (تابع call)\n",
    "\n",
    "3. و در نهایت یک تابع که ابعاد خروجی را محاسبه و تعیین می کند."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "لایه های تعریف شده باید از کلاس tf.keras.layers.Layers به ارث برده شده باشند."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### پیاده سازی لایه  Noisy Relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NoisyRelu(tf.keras.layers.Layer):\n",
    "    def __init__(self,**kwargs):\n",
    "        super(NoisyRelu,self).__init__(**kwargs)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "اگر به تعریف تابع فعال سازی noisy relu دقت کرده باشید می بینید که این لایه هیچ پارامتر آموزش پذیری ندارد. برای همین نیازی به تابع build نداریم.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def call(self,x):\n",
    "    tf.maximum(0, x + tf.random.normal())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_output_shape(self,input_shape):\n",
    "    return input_shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1, 1), dtype=float32, numpy=array([[0.16257451]], dtype=float32)>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.random.normal(shape = (1,1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "حالا می توانیم همه این ها را کنار هم قرار دهیم."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "class NoisyRelu(tf.keras.layers.Layer):\n",
    "    def __init__(self,**kwargs):\n",
    "        super(NoisyRelu,self).__init__(**kwargs)\n",
    "    def call(self,x):\n",
    "        return tf.maximum(0.0, x + tf.random.normal(shape = (1,1)))\n",
    "    def compute_output_shape(self,input_shape):\n",
    "        return input_shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### آموزش و امتحان لایه جدید ساخته شده"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Conv2D(filters = 15,kernel_size=3, input_shape = (28,28,1)))\n",
    "model.add(NoisyRelu())\n",
    "model.add(MaxPooling2D(pool_size=(2,2)))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(100,activation='relu'))\n",
    "model.add(Dense(10,activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_6 (Conv2D)            (None, 26, 26, 15)        150       \n",
      "_________________________________________________________________\n",
      "noisy_relu_6 (NoisyRelu)     (None, 26, 26, 15)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 13, 13, 15)        0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 2535)              0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 100)               253600    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 254,760\n",
      "Trainable params: 254,760\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### پیاده سازی لایه Attention"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "حالا نوبت این است که تابع build را تعریف کنیم.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build(self,input_shape):\n",
    "    self."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ScaledDotProductAttention(Layer):\n",
    "    \"\"\"\n",
    "        Implementation according to:\n",
    "            \"Attention is all you need\" by A Vaswani, N Shazeer, N Parmar (2017)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, return_attention=False, **kwargs):    \n",
    "        self._return_attention = return_attention\n",
    "        self.supports_masking = True\n",
    "        super(ScaledDotProductAttention, self).__init__(**kwargs)\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        self._validate_input_shape(input_shape)\n",
    "\n",
    "        if not self._return_attention:\n",
    "            return input_shape[-1]\n",
    "        else:\n",
    "            return [input_shape[-1], [input_shape[0][0], input_shape[0][1], input_shape[1][2]]]\n",
    "    \n",
    "    def _validate_input_shape(self, input_shape):\n",
    "        if len(input_shape) != 3:\n",
    "            raise ValueError(\"Layer received an input shape {0} but expected three inputs (Q, V, K).\".format(input_shape))\n",
    "        else:\n",
    "            if input_shape[0][0] != input_shape[1][0] or input_shape[1][0] != input_shape[2][0]:\n",
    "                raise ValueError(\"All three inputs (Q, V, K) have to have the same batch size; received batch sizes: {0}, {1}, {2}\".format(input_shape[0][0], input_shape[1][0], input_shape[2][0]))\n",
    "            if input_shape[0][1] != input_shape[1][1] or input_shape[1][1] != input_shape[2][1]:\n",
    "                raise ValueError(\"All three inputs (Q, V, K) have to have the same length; received lengths: {0}, {1}, {2}\".format(input_shape[0][0], input_shape[1][0], input_shape[2][0]))\n",
    "            if input_shape[0][2] != input_shape[1][2]:\n",
    "                raise ValueError(\"Input shapes of Q {0} and V {1} do not match.\".format(input_shape[0], input_shape[1]))\n",
    "    \n",
    "    def build(self, input_shape):\n",
    "        self._validate_input_shape(input_shape)\n",
    "        \n",
    "        super(ScaledDotProductAttention, self).build(input_shape)\n",
    "    \n",
    "    def call(self, x, mask=None):\n",
    "        q, k, v = x\n",
    "        d_k = q.shape.as_list()[2]\n",
    "\n",
    "        # in pure tensorflow:\n",
    "        # weights = tf.matmul(x_batch, tf.transpose(y_batch, perm=[0, 2, 1]))\n",
    "        # normalized_weights = tf.nn.softmax(weights/scaling)\n",
    "        # output = tf.matmul(normalized_weights, x_batch)\n",
    "        \n",
    "        weights = K.batch_dot(q,  k, axes=[2, 2])\n",
    "\n",
    "        if mask is not None:\n",
    "            # add mask weights\n",
    "            if isinstance(mask, (list, tuple)):\n",
    "                if len(mask) > 0:\n",
    "                    raise ValueError(\"mask can only be a Tensor or a list of length 1 containing a tensor.\")\n",
    "\n",
    "                mask = mask[0]\n",
    "\n",
    "            weights += -1e10*(1-mask)\n",
    "\n",
    "        normalized_weights = K.softmax(weights / np.sqrt(d_k))\n",
    "        output = K.batch_dot(normalized_weights, v)\n",
    "        \n",
    "        if self._return_attention:\n",
    "            return [output, normalized_weights]\n",
    "        else:\n",
    "            return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### منابع"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/zimmerrol/keras-utility-layer-collection/blob/master/kulc/layer_normalization.py\n",
    "https://github.com/zimmerrol/keras-utility-layer-collection\n",
    "    https://github.com/Zelgunn/CustomKerasLayers"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}