Jovian
⭐️
Sign In
In [1]:
import jovian
In [2]:
!pip install jovian --user
Requirement already satisfied: jovian in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (0.1.62) Requirement already satisfied: requests in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from jovian) (2.22.0) Requirement already satisfied: uuid in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from jovian) (1.30) Requirement already satisfied: certifi>=2017.4.17 in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from requests->jovian) (2019.6.16) Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from requests->jovian) (1.24.2) Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from requests->jovian) (3.0.4) Requirement already satisfied: idna<2.9,>=2.5 in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from requests->jovian) (2.6)
You are using pip version 18.1, however version 19.1.1 is available. You should consider upgrading via the 'python -m pip install --upgrade pip' command.

Assigment Definition:

In this assignement, we will compare different Deep Convolutional Neural Networks in terms of:

  • "number of parameters"
  • "inference time"
  • "performance".

You will be required to construct the following networks:

  1. AlexNet
  2. VGG-16
  3. Your custom Deep CNN (CNN_5x5_Net): use only 5x5 convolutional kernels (hint: in VGG-16, all convolutional kernels were 3x3)
  • Check the number of parameters vs inference time (by generating random noise image and feed-forward through the networks)

  • Explain the results: why one was faster than the another, what was the key point?

  • Add "Batch Normalization" and/or "Dropout" layers to the networks (AlexNet, VGG-16, Your custom CNN_5x5_Net)

  • Check how does inference time is changing after adding those layers Note: Framework: we will be using Keras in this assignment.

In [ ]:
import time 
import keras
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D,BatchNormalization
import numpy as np


def alexnet(input_data_shape=(224, 224, 3), number_of_classes=10):
    model = Sequential()

    # 1st Convolutional Layer
    model.add(Conv2D(filters=96, input_shape=input_data_shape, kernel_size=(11, 11), strides=(4, 4), padding='valid', activation='relu'))
    # Max Pooling
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))

    # 2nd Convolutional Layer
    model.add(Conv2D(filters=256, kernel_size=(11, 11), strides=(1, 1), padding='valid', activation='relu'))
    # Max Pooling
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))

    # 3rd Convolutional Layer
    model.add(Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), padding='valid', activation='relu'))

    # 4th Convolutional Layer
    model.add(Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), padding='valid', activation='relu'))

    # 5th Convolutional Layer
    model.add(Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='valid', activation='relu'))
    # Max Pooling
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))

    # Flatten the feature maps to pass them to Fully Connected Layers
    model.add(Flatten())

    # Fully Connected Layers
    model.add(Dense(4096, activation='relu'))
    model.add(Dense(4096, activation='relu'))
    model.add(Dense(number_of_classes, activation='softmax'))

    model.summary()
    return model
In [ ]:
def vgg_16(input_data_shape=(224, 224, 3), number_of_classes=10):
    model = Sequential()
    # Block 1
    model.add(Conv2D(filters=64, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(Conv2D(filters=64, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

    # Block 2
    model.add(Conv2D(filters=128, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(Conv2D(filters=128, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

    # Block 3
    model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

    # Block 4
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

    # Block 5
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

    # Flatten the feature maps to pass them to Fully Connected Layers
    model.add(Flatten())

    # fully connected layers
    model.add(Dense(4096, activation='relu'))
    model.add(Dense(4096, activation='relu'))
    model.add(Dense(number_of_classes, activation='softmax'))

    # Create model.
    model.summary()

    return model
In [ ]:
def custom_model_builder(input_data_shape=(224, 224, 3), number_of_classes=10):
    model = Sequential()
    #Block 1
    model.add(Conv2D(filters=32,kernel_size=(5,5),padding='same',activation='relu',input_shape = input_data_shape ))
    model.add(Conv2D(filters=32,kernel_size=(5,5),padding='same',activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding = 'same'))
    
    #Block 2
    model.add(Conv2D(filters=64,kernel_size=(5,5),padding='same',activation='relu'))
    model.add(Conv2D(filters=64,kernel_size=(5,5),padding='same',activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
    
    #Block 3
    model.add(Conv2D(filters=128,kernel_size=(5,5),padding='valid',activation='relu'))
    model.add(Conv2D(filters=128,kernel_size=(5,5),padding='same',activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
    
    #Block 4
    model.add(Conv2D(filters=256,kernel_size=(5, 5), padding='same', activation='relu'))
    model.add(Conv2D(filters=256,kernel_size=(5, 5), padding='valid', activation='relu'))
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))

    #flattening the model
    model.add(Flatten())
    
    #add dense layers
    model.add(Dense(2024,activation='relu'))
    model.add(Dense(512,activation='relu'))
    model.add(Dense(number_of_classes,activation='softmax'))
    
    return model

Set the inference hyper-parameters:

In [ ]:
batch_size = 128
num_classes = 10
num_of_training_iteration = 100

Set the input image shape/size

In [ ]:
input_data_shape = (224, 224, 3)

Build AlexNet & VGG-16 networks (default input image shape is 224x224x3)

In [6]:
alexnet_model = alexnet(input_data_shape=input_data_shape)
vgg16_model = vgg_16(input_data_shape=input_data_shape)
WARNING: Logging before flag parsing goes to stderr. W0710 01:51:57.929970 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead. W0710 01:51:57.971183 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead. W0710 01:51:57.977970 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead. W0710 01:51:58.010616 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.
_________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_1 (Conv2D) (None, 54, 54, 96) 34944 _________________________________________________________________ max_pooling2d_1 (MaxPooling2 (None, 27, 27, 96) 0 _________________________________________________________________ conv2d_2 (Conv2D) (None, 17, 17, 256) 2973952 _________________________________________________________________ max_pooling2d_2 (MaxPooling2 (None, 8, 8, 256) 0 _________________________________________________________________ conv2d_3 (Conv2D) (None, 6, 6, 384) 885120 _________________________________________________________________ conv2d_4 (Conv2D) (None, 4, 4, 384) 1327488 _________________________________________________________________ conv2d_5 (Conv2D) (None, 2, 2, 256) 884992 _________________________________________________________________ max_pooling2d_3 (MaxPooling2 (None, 1, 1, 256) 0 _________________________________________________________________ flatten_1 (Flatten) (None, 256) 0 _________________________________________________________________ dense_1 (Dense) (None, 4096) 1052672 _________________________________________________________________ dense_2 (Dense) (None, 4096) 16781312 _________________________________________________________________ dense_3 (Dense) (None, 10) 40970 ================================================================= Total params: 23,981,450 Trainable params: 23,981,450 Non-trainable params: 0 _________________________________________________________________ _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_6 (Conv2D) (None, 224, 224, 64) 1792 _________________________________________________________________ conv2d_7 (Conv2D) (None, 224, 224, 64) 36928 _________________________________________________________________ max_pooling2d_4 (MaxPooling2 (None, 112, 112, 64) 0 _________________________________________________________________ conv2d_8 (Conv2D) (None, 112, 112, 128) 73856 _________________________________________________________________ conv2d_9 (Conv2D) (None, 112, 112, 128) 147584 _________________________________________________________________ max_pooling2d_5 (MaxPooling2 (None, 56, 56, 128) 0 _________________________________________________________________ conv2d_10 (Conv2D) (None, 56, 56, 256) 295168 _________________________________________________________________ conv2d_11 (Conv2D) (None, 56, 56, 256) 590080 _________________________________________________________________ conv2d_12 (Conv2D) (None, 56, 56, 256) 590080 _________________________________________________________________ max_pooling2d_6 (MaxPooling2 (None, 28, 28, 256) 0 _________________________________________________________________ conv2d_13 (Conv2D) (None, 28, 28, 512) 1180160 _________________________________________________________________ conv2d_14 (Conv2D) (None, 28, 28, 512) 2359808 _________________________________________________________________ conv2d_15 (Conv2D) (None, 28, 28, 512) 2359808 _________________________________________________________________ max_pooling2d_7 (MaxPooling2 (None, 14, 14, 512) 0 _________________________________________________________________ conv2d_16 (Conv2D) (None, 14, 14, 512) 2359808 _________________________________________________________________ conv2d_17 (Conv2D) (None, 14, 14, 512) 2359808 _________________________________________________________________ conv2d_18 (Conv2D) (None, 14, 14, 512) 2359808 _________________________________________________________________ max_pooling2d_8 (MaxPooling2 (None, 7, 7, 512) 0 _________________________________________________________________ flatten_2 (Flatten) (None, 25088) 0 _________________________________________________________________ dense_4 (Dense) (None, 4096) 102764544 _________________________________________________________________ dense_5 (Dense) (None, 4096) 16781312 _________________________________________________________________ dense_6 (Dense) (None, 10) 40970 ================================================================= Total params: 134,301,514 Trainable params: 134,301,514 Non-trainable params: 0 _________________________________________________________________

Custom Model

In [7]:
custom_model = custom_model_builder(input_data_shape=input_data_shape)
custom_model.summary()
_________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_19 (Conv2D) (None, 224, 224, 32) 2432 _________________________________________________________________ conv2d_20 (Conv2D) (None, 224, 224, 32) 25632 _________________________________________________________________ max_pooling2d_9 (MaxPooling2 (None, 112, 112, 32) 0 _________________________________________________________________ conv2d_21 (Conv2D) (None, 112, 112, 64) 51264 _________________________________________________________________ conv2d_22 (Conv2D) (None, 112, 112, 64) 102464 _________________________________________________________________ max_pooling2d_10 (MaxPooling (None, 56, 56, 64) 0 _________________________________________________________________ conv2d_23 (Conv2D) (None, 52, 52, 128) 204928 _________________________________________________________________ conv2d_24 (Conv2D) (None, 52, 52, 128) 409728 _________________________________________________________________ max_pooling2d_11 (MaxPooling (None, 26, 26, 128) 0 _________________________________________________________________ conv2d_25 (Conv2D) (None, 26, 26, 256) 819456 _________________________________________________________________ conv2d_26 (Conv2D) (None, 22, 22, 256) 1638656 _________________________________________________________________ max_pooling2d_12 (MaxPooling (None, 11, 11, 256) 0 _________________________________________________________________ flatten_3 (Flatten) (None, 30976) 0 _________________________________________________________________ dense_7 (Dense) (None, 2024) 62697448 _________________________________________________________________ dense_8 (Dense) (None, 512) 1036800 _________________________________________________________________ dense_9 (Dense) (None, 10) 5130 ================================================================= Total params: 66,993,938 Trainable params: 66,993,938 Non-trainable params: 0 _________________________________________________________________

Train the networks on MNIST dataset:

In [12]:
alexnet_model.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
vgg16_model.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
custom_model.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])

alexnet_inference_time = []
vgg16_inference_time = []
custom_inference_time = []


# dummy tensor to check the inference time of each network
x_test = np.random.rand(batch_size, input_data_shape[0], input_data_shape[1], input_data_shape[2])

for _ in range(num_of_training_iteration):
    alexnet_inference_start = time.time()
    alexnet_inference = alexnet_model.predict_classes(x_test)
    alexnet_inference_finish = time.time()
    alexnet_inference_time.append(alexnet_inference_finish - alexnet_inference_start)

print("Average Inference time for AlexNet: {}".format(np.mean(alexnet_inference_time)))
    
for _ in range(num_of_training_iteration):
    vgg16_inference_start = time.time()
    vgg_inference = vgg16_model.predict_classes(x_test)
    vgg16_inference_finish = time.time()
    vgg16_inference_time.append(vgg16_inference_finish - vgg16_inference_start)
    
print("Average Inference time for VGG-16: {}".format(np.mean(vgg16_inference_time)))    
    
    
for _ in range(num_of_training_iteration):
    custom_inference_start = time.time()
    custom_inference = custom_model.predict_classes(x_test)
    custom_inference_finish = time.time()
    custom_inference_time.append(custom_inference_finish - custom_inference_start)
    
print("Average Inference time for Custom Model: {}".format(np.mean(custom_inference_time)))
Average Inference time for AlexNet: 0.18639827251434327 Average Inference time for VGG-16: 1.2100702261924743 Average Inference time for Custom Model: 0.5777664661407471

Key Points

  • AlexNet has lowest parameters (23.9 Millions),Custom Model has 67 Million parameters and VGG-16 has highest parameters(134 Millions).

  • Average Inference time is as same order as the number of Parameters.

  • Custom model has approx half number of parameters(67 Million) than VGG-16 (134 Millions).Average inference time of custom model is also approximately half than VGG-16 model.

Conclusion - Inference Time is approximately directly proportional to Number of parameters.

In [ ]:
 
In [ ]:
 
In [ ]:
 

Let's add batch normalization and dropout layers

Note- Adding batch normalization before activation. It is matter of debate when to add batch normalization layers. I added batch normalization before applying activation function. Andrew Ng says in his course add batch normalization before activation and Jeremy Howard says after activation.

Andrew Ng

https://www.youtube.com/watch?v=tNIpEZLv_eg&t=471s

Jeremy Howard

https://forums.fast.ai/t/questions-about-batch-normalization/230/3

Great Discussion on Reddit.

https://www.reddit.com/r/MachineLearning/comments/67gonq/d_batch_normalization_before_or_after_relu/

In [ ]:
def alexnet_do_bn(input_data_shape=(224, 224, 3), number_of_classes=10):
    model = Sequential()

    # 1st Convolutional Layer
    model.add(Conv2D(filters=96, input_shape=input_data_shape, kernel_size=(11, 11), strides=(4, 4), padding='valid', activation='relu'))
    # Max Pooling
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
    

    # 2nd Convolutional Layer
    model.add(Conv2D(filters=256, kernel_size=(11, 11), strides=(1, 1), padding='valid'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    # Max Pooling
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))

    #dropout
    model.add(Dropout(0.25))
    
    # 3rd Convolutional Layer
    model.add(Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), padding='valid'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))

    # 4th Convolutional Layer
    model.add(Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), padding='valid'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    #dropout
    model.add(Dropout(0.25))
    

    # 5th Convolutional Layer
    model.add(Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='valid'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    # Max Pooling
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))

    # Flatten the feature maps to pass them to Fully Connected Layers
    model.add(Flatten())

    # Fully Connected Layers
    model.add(Dense(4096))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Dense(4096))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    #dropout
    model.add(Dropout(0.5))
    model.add(Dense(number_of_classes, activation='softmax'))

    model.summary()
    return model
In [ ]:
def vgg_16_do_bn(input_data_shape=(224, 224, 3), number_of_classes=10):
    model = Sequential()
    # Block 1
    model.add(Conv2D(filters=64, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=64, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

    # Block 2
    model.add(Conv2D(filters=128, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=128, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

    #dropout
    model.add(Dropout(0.25))

    # Block 3
    model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

    # Block 4
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
    
    #dropout
    model.add(Dropout(0.25))

    
    # Block 5
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))

    # Flatten the feature maps to pass them to Fully Connected Layers
    model.add(Flatten())

    # fully connected layers
    model.add(Dense(4096))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Dense(4096))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    #dropout
    model.add(Dropout(0.5))

    model.add(Dense(number_of_classes, activation='softmax'))

    # Create model.
    model.summary()

    return model
In [ ]:
def custom_model_builder_do_bn(input_data_shape=(224, 224, 3), number_of_classes=10):
    model = Sequential()
    #Block 1
    model.add(Conv2D(filters=32,kernel_size=(5,5),padding='same',input_shape = input_data_shape ))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=32,kernel_size=(5,5),padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding = 'same'))
    
    #Block 2
    model.add(Conv2D(filters=64,kernel_size=(5,5),padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=64,kernel_size=(5,5),padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
    
    #dropout
    model.add(Dropout(0.25))

    
    #Block 3
    model.add(Conv2D(filters=128,kernel_size=(5,5),padding='valid'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=128,kernel_size=(5,5),padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
    
    #Block 4
    model.add(Conv2D(filters=256,kernel_size=(5, 5), padding='same'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Conv2D(filters=256,kernel_size=(5, 5), padding='valid'))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))

    #dropout
    model.add(Dropout(0.25))

    
    #flattening the model
    model.add(Flatten())
    
    #add dense layers
    model.add(Dense(2024))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    model.add(Dense(512))
    
    #adding batch normalization
    model.add(BatchNormalization())
    model.add(Activation('relu'))
    
    
    #dropout
    model.add(Dropout(0.5))

    model.add(Dense(number_of_classes,activation='softmax'))
    
    return model
In [16]:
alexnet_model_do_bn = alexnet_do_bn(input_data_shape=input_data_shape)
vgg16_model_do_bn = vgg_16_do_bn(input_data_shape=input_data_shape)
W0710 02:09:23.779240 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1834: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead. W0710 02:09:23.872189 139695120390016 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version. Instructions for updating: Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
_________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_27 (Conv2D) (None, 54, 54, 96) 34944 _________________________________________________________________ max_pooling2d_13 (MaxPooling (None, 27, 27, 96) 0 _________________________________________________________________ conv2d_28 (Conv2D) (None, 17, 17, 256) 2973952 _________________________________________________________________ batch_normalization_1 (Batch (None, 17, 17, 256) 1024 _________________________________________________________________ activation_1 (Activation) (None, 17, 17, 256) 0 _________________________________________________________________ max_pooling2d_14 (MaxPooling (None, 8, 8, 256) 0 _________________________________________________________________ dropout_1 (Dropout) (None, 8, 8, 256) 0 _________________________________________________________________ conv2d_29 (Conv2D) (None, 6, 6, 384) 885120 _________________________________________________________________ batch_normalization_2 (Batch (None, 6, 6, 384) 1536 _________________________________________________________________ activation_2 (Activation) (None, 6, 6, 384) 0 _________________________________________________________________ conv2d_30 (Conv2D) (None, 4, 4, 384) 1327488 _________________________________________________________________ batch_normalization_3 (Batch (None, 4, 4, 384) 1536 _________________________________________________________________ activation_3 (Activation) (None, 4, 4, 384) 0 _________________________________________________________________ dropout_2 (Dropout) (None, 4, 4, 384) 0 _________________________________________________________________ conv2d_31 (Conv2D) (None, 2, 2, 256) 884992 _________________________________________________________________ batch_normalization_4 (Batch (None, 2, 2, 256) 1024 _________________________________________________________________ activation_4 (Activation) (None, 2, 2, 256) 0 _________________________________________________________________ max_pooling2d_15 (MaxPooling (None, 1, 1, 256) 0 _________________________________________________________________ flatten_4 (Flatten) (None, 256) 0 _________________________________________________________________ dense_10 (Dense) (None, 4096) 1052672 _________________________________________________________________ batch_normalization_5 (Batch (None, 4096) 16384 _________________________________________________________________ activation_5 (Activation) (None, 4096) 0 _________________________________________________________________ dense_11 (Dense) (None, 4096) 16781312 _________________________________________________________________ batch_normalization_6 (Batch (None, 4096) 16384 _________________________________________________________________ activation_6 (Activation) (None, 4096) 0 _________________________________________________________________ dropout_3 (Dropout) (None, 4096) 0 _________________________________________________________________ dense_12 (Dense) (None, 10) 40970 ================================================================= Total params: 24,019,338 Trainable params: 24,000,394 Non-trainable params: 18,944 _________________________________________________________________ _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_32 (Conv2D) (None, 224, 224, 64) 1792 _________________________________________________________________ batch_normalization_7 (Batch (None, 224, 224, 64) 256 _________________________________________________________________ activation_7 (Activation) (None, 224, 224, 64) 0 _________________________________________________________________ conv2d_33 (Conv2D) (None, 224, 224, 64) 36928 _________________________________________________________________ batch_normalization_8 (Batch (None, 224, 224, 64) 256 _________________________________________________________________ activation_8 (Activation) (None, 224, 224, 64) 0 _________________________________________________________________ max_pooling2d_16 (MaxPooling (None, 112, 112, 64) 0 _________________________________________________________________ conv2d_34 (Conv2D) (None, 112, 112, 128) 73856 _________________________________________________________________ batch_normalization_9 (Batch (None, 112, 112, 128) 512 _________________________________________________________________ activation_9 (Activation) (None, 112, 112, 128) 0 _________________________________________________________________ conv2d_35 (Conv2D) (None, 112, 112, 128) 147584 _________________________________________________________________ batch_normalization_10 (Batc (None, 112, 112, 128) 512 _________________________________________________________________ activation_10 (Activation) (None, 112, 112, 128) 0 _________________________________________________________________ max_pooling2d_17 (MaxPooling (None, 56, 56, 128) 0 _________________________________________________________________ dropout_4 (Dropout) (None, 56, 56, 128) 0 _________________________________________________________________ conv2d_36 (Conv2D) (None, 56, 56, 256) 295168 _________________________________________________________________ batch_normalization_11 (Batc (None, 56, 56, 256) 1024 _________________________________________________________________ activation_11 (Activation) (None, 56, 56, 256) 0 _________________________________________________________________ conv2d_37 (Conv2D) (None, 56, 56, 256) 590080 _________________________________________________________________ batch_normalization_12 (Batc (None, 56, 56, 256) 1024 _________________________________________________________________ activation_12 (Activation) (None, 56, 56, 256) 0 _________________________________________________________________ conv2d_38 (Conv2D) (None, 56, 56, 256) 590080 _________________________________________________________________ batch_normalization_13 (Batc (None, 56, 56, 256) 1024 _________________________________________________________________ activation_13 (Activation) (None, 56, 56, 256) 0 _________________________________________________________________ max_pooling2d_18 (MaxPooling (None, 28, 28, 256) 0 _________________________________________________________________ conv2d_39 (Conv2D) (None, 28, 28, 512) 1180160 _________________________________________________________________ batch_normalization_14 (Batc (None, 28, 28, 512) 2048 _________________________________________________________________ activation_14 (Activation) (None, 28, 28, 512) 0 _________________________________________________________________ conv2d_40 (Conv2D) (None, 28, 28, 512) 2359808 _________________________________________________________________ batch_normalization_15 (Batc (None, 28, 28, 512) 2048 _________________________________________________________________ activation_15 (Activation) (None, 28, 28, 512) 0 _________________________________________________________________ conv2d_41 (Conv2D) (None, 28, 28, 512) 2359808 _________________________________________________________________ batch_normalization_16 (Batc (None, 28, 28, 512) 2048 _________________________________________________________________ activation_16 (Activation) (None, 28, 28, 512) 0 _________________________________________________________________ max_pooling2d_19 (MaxPooling (None, 14, 14, 512) 0 _________________________________________________________________ dropout_5 (Dropout) (None, 14, 14, 512) 0 _________________________________________________________________ conv2d_42 (Conv2D) (None, 14, 14, 512) 2359808 _________________________________________________________________ batch_normalization_17 (Batc (None, 14, 14, 512) 2048 _________________________________________________________________ activation_17 (Activation) (None, 14, 14, 512) 0 _________________________________________________________________ conv2d_43 (Conv2D) (None, 14, 14, 512) 2359808 _________________________________________________________________ batch_normalization_18 (Batc (None, 14, 14, 512) 2048 _________________________________________________________________ activation_18 (Activation) (None, 14, 14, 512) 0 _________________________________________________________________ conv2d_44 (Conv2D) (None, 14, 14, 512) 2359808 _________________________________________________________________ batch_normalization_19 (Batc (None, 14, 14, 512) 2048 _________________________________________________________________ activation_19 (Activation) (None, 14, 14, 512) 0 _________________________________________________________________ max_pooling2d_20 (MaxPooling (None, 7, 7, 512) 0 _________________________________________________________________ flatten_5 (Flatten) (None, 25088) 0 _________________________________________________________________ dense_13 (Dense) (None, 4096) 102764544 _________________________________________________________________ batch_normalization_20 (Batc (None, 4096) 16384 _________________________________________________________________ activation_20 (Activation) (None, 4096) 0 _________________________________________________________________ dense_14 (Dense) (None, 4096) 16781312 _________________________________________________________________ batch_normalization_21 (Batc (None, 4096) 16384 _________________________________________________________________ activation_21 (Activation) (None, 4096) 0 _________________________________________________________________ dropout_6 (Dropout) (None, 4096) 0 _________________________________________________________________ dense_15 (Dense) (None, 10) 40970 ================================================================= Total params: 134,351,178 Trainable params: 134,326,346 Non-trainable params: 24,832 _________________________________________________________________
In [17]:
custom_model_do_bn = custom_model_builder_do_bn(input_data_shape=input_data_shape)
custom_model_do_bn.summary()
_________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_45 (Conv2D) (None, 224, 224, 32) 2432 _________________________________________________________________ batch_normalization_22 (Batc (None, 224, 224, 32) 128 _________________________________________________________________ activation_22 (Activation) (None, 224, 224, 32) 0 _________________________________________________________________ conv2d_46 (Conv2D) (None, 224, 224, 32) 25632 _________________________________________________________________ batch_normalization_23 (Batc (None, 224, 224, 32) 128 _________________________________________________________________ activation_23 (Activation) (None, 224, 224, 32) 0 _________________________________________________________________ max_pooling2d_21 (MaxPooling (None, 112, 112, 32) 0 _________________________________________________________________ conv2d_47 (Conv2D) (None, 112, 112, 64) 51264 _________________________________________________________________ batch_normalization_24 (Batc (None, 112, 112, 64) 256 _________________________________________________________________ activation_24 (Activation) (None, 112, 112, 64) 0 _________________________________________________________________ conv2d_48 (Conv2D) (None, 112, 112, 64) 102464 _________________________________________________________________ batch_normalization_25 (Batc (None, 112, 112, 64) 256 _________________________________________________________________ activation_25 (Activation) (None, 112, 112, 64) 0 _________________________________________________________________ max_pooling2d_22 (MaxPooling (None, 56, 56, 64) 0 _________________________________________________________________ dropout_7 (Dropout) (None, 56, 56, 64) 0 _________________________________________________________________ conv2d_49 (Conv2D) (None, 52, 52, 128) 204928 _________________________________________________________________ batch_normalization_26 (Batc (None, 52, 52, 128) 512 _________________________________________________________________ activation_26 (Activation) (None, 52, 52, 128) 0 _________________________________________________________________ conv2d_50 (Conv2D) (None, 52, 52, 128) 409728 _________________________________________________________________ batch_normalization_27 (Batc (None, 52, 52, 128) 512 _________________________________________________________________ activation_27 (Activation) (None, 52, 52, 128) 0 _________________________________________________________________ max_pooling2d_23 (MaxPooling (None, 26, 26, 128) 0 _________________________________________________________________ conv2d_51 (Conv2D) (None, 26, 26, 256) 819456 _________________________________________________________________ batch_normalization_28 (Batc (None, 26, 26, 256) 1024 _________________________________________________________________ activation_28 (Activation) (None, 26, 26, 256) 0 _________________________________________________________________ conv2d_52 (Conv2D) (None, 22, 22, 256) 1638656 _________________________________________________________________ batch_normalization_29 (Batc (None, 22, 22, 256) 1024 _________________________________________________________________ activation_29 (Activation) (None, 22, 22, 256) 0 _________________________________________________________________ max_pooling2d_24 (MaxPooling (None, 11, 11, 256) 0 _________________________________________________________________ dropout_8 (Dropout) (None, 11, 11, 256) 0 _________________________________________________________________ flatten_6 (Flatten) (None, 30976) 0 _________________________________________________________________ dense_16 (Dense) (None, 2024) 62697448 _________________________________________________________________ batch_normalization_30 (Batc (None, 2024) 8096 _________________________________________________________________ activation_30 (Activation) (None, 2024) 0 _________________________________________________________________ dense_17 (Dense) (None, 512) 1036800 _________________________________________________________________ batch_normalization_31 (Batc (None, 512) 2048 _________________________________________________________________ activation_31 (Activation) (None, 512) 0 _________________________________________________________________ dropout_9 (Dropout) (None, 512) 0 _________________________________________________________________ dense_18 (Dense) (None, 10) 5130 ================================================================= Total params: 67,007,922 Trainable params: 67,000,930 Non-trainable params: 6,992 _________________________________________________________________
In [18]:
custom_model_do_bn = custom_model_builder_do_bn(input_data_shape=input_data_shape)
custom_model_do_bn.summary()
_________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_53 (Conv2D) (None, 224, 224, 32) 2432 _________________________________________________________________ batch_normalization_32 (Batc (None, 224, 224, 32) 128 _________________________________________________________________ activation_32 (Activation) (None, 224, 224, 32) 0 _________________________________________________________________ conv2d_54 (Conv2D) (None, 224, 224, 32) 25632 _________________________________________________________________ batch_normalization_33 (Batc (None, 224, 224, 32) 128 _________________________________________________________________ activation_33 (Activation) (None, 224, 224, 32) 0 _________________________________________________________________ max_pooling2d_25 (MaxPooling (None, 112, 112, 32) 0 _________________________________________________________________ conv2d_55 (Conv2D) (None, 112, 112, 64) 51264 _________________________________________________________________ batch_normalization_34 (Batc (None, 112, 112, 64) 256 _________________________________________________________________ activation_34 (Activation) (None, 112, 112, 64) 0 _________________________________________________________________ conv2d_56 (Conv2D) (None, 112, 112, 64) 102464 _________________________________________________________________ batch_normalization_35 (Batc (None, 112, 112, 64) 256 _________________________________________________________________ activation_35 (Activation) (None, 112, 112, 64) 0 _________________________________________________________________ max_pooling2d_26 (MaxPooling (None, 56, 56, 64) 0 _________________________________________________________________ dropout_10 (Dropout) (None, 56, 56, 64) 0 _________________________________________________________________ conv2d_57 (Conv2D) (None, 52, 52, 128) 204928 _________________________________________________________________ batch_normalization_36 (Batc (None, 52, 52, 128) 512 _________________________________________________________________ activation_36 (Activation) (None, 52, 52, 128) 0 _________________________________________________________________ conv2d_58 (Conv2D) (None, 52, 52, 128) 409728 _________________________________________________________________ batch_normalization_37 (Batc (None, 52, 52, 128) 512 _________________________________________________________________ activation_37 (Activation) (None, 52, 52, 128) 0 _________________________________________________________________ max_pooling2d_27 (MaxPooling (None, 26, 26, 128) 0 _________________________________________________________________ conv2d_59 (Conv2D) (None, 26, 26, 256) 819456 _________________________________________________________________ batch_normalization_38 (Batc (None, 26, 26, 256) 1024 _________________________________________________________________ activation_38 (Activation) (None, 26, 26, 256) 0 _________________________________________________________________ conv2d_60 (Conv2D) (None, 22, 22, 256) 1638656 _________________________________________________________________ batch_normalization_39 (Batc (None, 22, 22, 256) 1024 _________________________________________________________________ activation_39 (Activation) (None, 22, 22, 256) 0 _________________________________________________________________ max_pooling2d_28 (MaxPooling (None, 11, 11, 256) 0 _________________________________________________________________ dropout_11 (Dropout) (None, 11, 11, 256) 0 _________________________________________________________________ flatten_7 (Flatten) (None, 30976) 0 _________________________________________________________________ dense_19 (Dense) (None, 2024) 62697448 _________________________________________________________________ batch_normalization_40 (Batc (None, 2024) 8096 _________________________________________________________________ activation_40 (Activation) (None, 2024) 0 _________________________________________________________________ dense_20 (Dense) (None, 512) 1036800 _________________________________________________________________ batch_normalization_41 (Batc (None, 512) 2048 _________________________________________________________________ activation_41 (Activation) (None, 512) 0 _________________________________________________________________ dropout_12 (Dropout) (None, 512) 0 _________________________________________________________________ dense_21 (Dense) (None, 10) 5130 ================================================================= Total params: 67,007,922 Trainable params: 67,000,930 Non-trainable params: 6,992 _________________________________________________________________
In [20]:
alexnet_model_do_bn.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
vgg16_model_do_bn.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
custom_model_do_bn.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])


alexnet_inference_time_do_bn = []
vgg16_inference_time_do_bn = []
custom_inference_time_do_bn = []


# dummy tensor to check the inference time of each network
x_test = np.random.rand(batch_size, input_data_shape[0], input_data_shape[1], input_data_shape[2])

for _ in range(num_of_training_iteration):
    alexnet_inference_start = time.time()
    alexnet_inference = alexnet_model_do_bn.predict_classes(x_test)
    alexnet_inference_finish = time.time()
    alexnet_inference_time_do_bn.append(alexnet_inference_finish - alexnet_inference_start)

print("Average Inference time for AlexNet: {}".format(np.mean(alexnet_inference_time_do_bn)))
    
for _ in range(num_of_training_iteration):
    vgg16_inference_start = time.time()
    vgg_inference = vgg16_model_do_bn.predict_classes(x_test)
    vgg16_inference_finish = time.time()
    vgg16_inference_time_do_bn.append(vgg16_inference_finish - vgg16_inference_start)
    
print("Average Inference time for VGG-16: {}".format(np.mean(vgg16_inference_time)))    
    
    
for _ in range(num_of_training_iteration):
    custom_inference_start = time.time()
    custom_inference = custom_model_do_bn.predict_classes(x_test)
    custom_inference_finish = time.time()
    custom_inference_time_do_bn.append(custom_inference_finish - custom_inference_start)
    


print("Average Inference time for Custom Model: {}".format(np.mean(custom_inference_time)))
Average Inference time for AlexNet: 0.204864182472229 Average Inference time for VGG-16: 1.2100702261924743 Average Inference time for Custom Model: 0.5777664661407471

Conclusion

Average Inference time doesn't change much after adding batch normalization and dropout, because in inference time when we are doing batch normalization, we are just adding and multiplying outputs by scalers. In AlexNet inference time increased little bit because it has less parameters, but for other models the batch normalization parameters are very less, compared to other model parameters.**

In [ ]:
jovian.commit()
[jovian] Saving notebook..
In [ ]: