import jovian
!pip install jovian --user
Requirement already satisfied: jovian in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (0.1.62)
Requirement already satisfied: requests in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from jovian) (2.22.0)
Requirement already satisfied: uuid in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from jovian) (1.30)
Requirement already satisfied: certifi>=2017.4.17 in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from requests->jovian) (2019.6.16)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from requests->jovian) (1.24.2)
Requirement already satisfied: chardet<3.1.0,>=3.0.2 in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from requests->jovian) (3.0.4)
Requirement already satisfied: idna<2.9,>=2.5 in c:\users\neeraj\anaconda3\envs\tf\lib\site-packages (from requests->jovian) (2.6)
You are using pip version 18.1, however version 19.1.1 is available.
You should consider upgrading via the 'python -m pip install --upgrade pip' command.
In this assignement, we will compare different Deep Convolutional Neural Networks in terms of:
You will be required to construct the following networks:
Check the number of parameters vs inference time (by generating random noise image and feed-forward through the networks)
Explain the results: why one was faster than the another, what was the key point?
Add "Batch Normalization" and/or "Dropout" layers to the networks (AlexNet, VGG-16, Your custom CNN_5x5_Net)
Check how does inference time is changing after adding those layers Note: Framework: we will be using Keras in this assignment.
import time
import keras
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout, Flatten, Conv2D, MaxPooling2D,BatchNormalization
import numpy as np
def alexnet(input_data_shape=(224, 224, 3), number_of_classes=10):
model = Sequential()
# 1st Convolutional Layer
model.add(Conv2D(filters=96, input_shape=input_data_shape, kernel_size=(11, 11), strides=(4, 4), padding='valid', activation='relu'))
# Max Pooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
# 2nd Convolutional Layer
model.add(Conv2D(filters=256, kernel_size=(11, 11), strides=(1, 1), padding='valid', activation='relu'))
# Max Pooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
# 3rd Convolutional Layer
model.add(Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), padding='valid', activation='relu'))
# 4th Convolutional Layer
model.add(Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), padding='valid', activation='relu'))
# 5th Convolutional Layer
model.add(Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='valid', activation='relu'))
# Max Pooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
# Flatten the feature maps to pass them to Fully Connected Layers
model.add(Flatten())
# Fully Connected Layers
model.add(Dense(4096, activation='relu'))
model.add(Dense(4096, activation='relu'))
model.add(Dense(number_of_classes, activation='softmax'))
model.summary()
return model
def vgg_16(input_data_shape=(224, 224, 3), number_of_classes=10):
model = Sequential()
# Block 1
model.add(Conv2D(filters=64, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(Conv2D(filters=64, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Block 2
model.add(Conv2D(filters=128, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(Conv2D(filters=128, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Block 3
model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Block 4
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Block 5
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Flatten the feature maps to pass them to Fully Connected Layers
model.add(Flatten())
# fully connected layers
model.add(Dense(4096, activation='relu'))
model.add(Dense(4096, activation='relu'))
model.add(Dense(number_of_classes, activation='softmax'))
# Create model.
model.summary()
return model
def custom_model_builder(input_data_shape=(224, 224, 3), number_of_classes=10):
model = Sequential()
#Block 1
model.add(Conv2D(filters=32,kernel_size=(5,5),padding='same',activation='relu',input_shape = input_data_shape ))
model.add(Conv2D(filters=32,kernel_size=(5,5),padding='same',activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding = 'same'))
#Block 2
model.add(Conv2D(filters=64,kernel_size=(5,5),padding='same',activation='relu'))
model.add(Conv2D(filters=64,kernel_size=(5,5),padding='same',activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
#Block 3
model.add(Conv2D(filters=128,kernel_size=(5,5),padding='valid',activation='relu'))
model.add(Conv2D(filters=128,kernel_size=(5,5),padding='same',activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
#Block 4
model.add(Conv2D(filters=256,kernel_size=(5, 5), padding='same', activation='relu'))
model.add(Conv2D(filters=256,kernel_size=(5, 5), padding='valid', activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
#flattening the model
model.add(Flatten())
#add dense layers
model.add(Dense(2024,activation='relu'))
model.add(Dense(512,activation='relu'))
model.add(Dense(number_of_classes,activation='softmax'))
return model
batch_size = 128
num_classes = 10
num_of_training_iteration = 100
input_data_shape = (224, 224, 3)
alexnet_model = alexnet(input_data_shape=input_data_shape)
vgg16_model = vgg_16(input_data_shape=input_data_shape)
WARNING: Logging before flag parsing goes to stderr.
W0710 01:51:57.929970 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.
W0710 01:51:57.971183 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.
W0710 01:51:57.977970 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.
W0710 01:51:58.010616 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_1 (Conv2D) (None, 54, 54, 96) 34944
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 27, 27, 96) 0
_________________________________________________________________
conv2d_2 (Conv2D) (None, 17, 17, 256) 2973952
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 8, 8, 256) 0
_________________________________________________________________
conv2d_3 (Conv2D) (None, 6, 6, 384) 885120
_________________________________________________________________
conv2d_4 (Conv2D) (None, 4, 4, 384) 1327488
_________________________________________________________________
conv2d_5 (Conv2D) (None, 2, 2, 256) 884992
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 1, 1, 256) 0
_________________________________________________________________
flatten_1 (Flatten) (None, 256) 0
_________________________________________________________________
dense_1 (Dense) (None, 4096) 1052672
_________________________________________________________________
dense_2 (Dense) (None, 4096) 16781312
_________________________________________________________________
dense_3 (Dense) (None, 10) 40970
=================================================================
Total params: 23,981,450
Trainable params: 23,981,450
Non-trainable params: 0
_________________________________________________________________
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_6 (Conv2D) (None, 224, 224, 64) 1792
_________________________________________________________________
conv2d_7 (Conv2D) (None, 224, 224, 64) 36928
_________________________________________________________________
max_pooling2d_4 (MaxPooling2 (None, 112, 112, 64) 0
_________________________________________________________________
conv2d_8 (Conv2D) (None, 112, 112, 128) 73856
_________________________________________________________________
conv2d_9 (Conv2D) (None, 112, 112, 128) 147584
_________________________________________________________________
max_pooling2d_5 (MaxPooling2 (None, 56, 56, 128) 0
_________________________________________________________________
conv2d_10 (Conv2D) (None, 56, 56, 256) 295168
_________________________________________________________________
conv2d_11 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
conv2d_12 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
max_pooling2d_6 (MaxPooling2 (None, 28, 28, 256) 0
_________________________________________________________________
conv2d_13 (Conv2D) (None, 28, 28, 512) 1180160
_________________________________________________________________
conv2d_14 (Conv2D) (None, 28, 28, 512) 2359808
_________________________________________________________________
conv2d_15 (Conv2D) (None, 28, 28, 512) 2359808
_________________________________________________________________
max_pooling2d_7 (MaxPooling2 (None, 14, 14, 512) 0
_________________________________________________________________
conv2d_16 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
conv2d_17 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
conv2d_18 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
max_pooling2d_8 (MaxPooling2 (None, 7, 7, 512) 0
_________________________________________________________________
flatten_2 (Flatten) (None, 25088) 0
_________________________________________________________________
dense_4 (Dense) (None, 4096) 102764544
_________________________________________________________________
dense_5 (Dense) (None, 4096) 16781312
_________________________________________________________________
dense_6 (Dense) (None, 10) 40970
=================================================================
Total params: 134,301,514
Trainable params: 134,301,514
Non-trainable params: 0
_________________________________________________________________
custom_model = custom_model_builder(input_data_shape=input_data_shape)
custom_model.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_19 (Conv2D) (None, 224, 224, 32) 2432
_________________________________________________________________
conv2d_20 (Conv2D) (None, 224, 224, 32) 25632
_________________________________________________________________
max_pooling2d_9 (MaxPooling2 (None, 112, 112, 32) 0
_________________________________________________________________
conv2d_21 (Conv2D) (None, 112, 112, 64) 51264
_________________________________________________________________
conv2d_22 (Conv2D) (None, 112, 112, 64) 102464
_________________________________________________________________
max_pooling2d_10 (MaxPooling (None, 56, 56, 64) 0
_________________________________________________________________
conv2d_23 (Conv2D) (None, 52, 52, 128) 204928
_________________________________________________________________
conv2d_24 (Conv2D) (None, 52, 52, 128) 409728
_________________________________________________________________
max_pooling2d_11 (MaxPooling (None, 26, 26, 128) 0
_________________________________________________________________
conv2d_25 (Conv2D) (None, 26, 26, 256) 819456
_________________________________________________________________
conv2d_26 (Conv2D) (None, 22, 22, 256) 1638656
_________________________________________________________________
max_pooling2d_12 (MaxPooling (None, 11, 11, 256) 0
_________________________________________________________________
flatten_3 (Flatten) (None, 30976) 0
_________________________________________________________________
dense_7 (Dense) (None, 2024) 62697448
_________________________________________________________________
dense_8 (Dense) (None, 512) 1036800
_________________________________________________________________
dense_9 (Dense) (None, 10) 5130
=================================================================
Total params: 66,993,938
Trainable params: 66,993,938
Non-trainable params: 0
_________________________________________________________________
alexnet_model.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
vgg16_model.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
custom_model.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
alexnet_inference_time = []
vgg16_inference_time = []
custom_inference_time = []
# dummy tensor to check the inference time of each network
x_test = np.random.rand(batch_size, input_data_shape[0], input_data_shape[1], input_data_shape[2])
for _ in range(num_of_training_iteration):
alexnet_inference_start = time.time()
alexnet_inference = alexnet_model.predict_classes(x_test)
alexnet_inference_finish = time.time()
alexnet_inference_time.append(alexnet_inference_finish - alexnet_inference_start)
print("Average Inference time for AlexNet: {}".format(np.mean(alexnet_inference_time)))
for _ in range(num_of_training_iteration):
vgg16_inference_start = time.time()
vgg_inference = vgg16_model.predict_classes(x_test)
vgg16_inference_finish = time.time()
vgg16_inference_time.append(vgg16_inference_finish - vgg16_inference_start)
print("Average Inference time for VGG-16: {}".format(np.mean(vgg16_inference_time)))
for _ in range(num_of_training_iteration):
custom_inference_start = time.time()
custom_inference = custom_model.predict_classes(x_test)
custom_inference_finish = time.time()
custom_inference_time.append(custom_inference_finish - custom_inference_start)
print("Average Inference time for Custom Model: {}".format(np.mean(custom_inference_time)))
Average Inference time for AlexNet: 0.18639827251434327
Average Inference time for VGG-16: 1.2100702261924743
Average Inference time for Custom Model: 0.5777664661407471
AlexNet has lowest parameters (23.9 Millions),Custom Model has 67 Million parameters and VGG-16 has highest parameters(134 Millions).
Average Inference time is as same order as the number of Parameters.
Custom model has approx half number of parameters(67 Million) than VGG-16 (134 Millions).Average inference time of custom model is also approximately half than VGG-16 model.
Note- Adding batch normalization before activation. It is matter of debate when to add batch normalization layers. I added batch normalization before applying activation function. Andrew Ng says in his course add batch normalization before activation and Jeremy Howard says after activation.
Andrew Ng
https://www.youtube.com/watch?v=tNIpEZLv_eg&t=471s
Jeremy Howard
https://forums.fast.ai/t/questions-about-batch-normalization/230/3
Great Discussion on Reddit.
https://www.reddit.com/r/MachineLearning/comments/67gonq/d_batch_normalization_before_or_after_relu/
def alexnet_do_bn(input_data_shape=(224, 224, 3), number_of_classes=10):
model = Sequential()
# 1st Convolutional Layer
model.add(Conv2D(filters=96, input_shape=input_data_shape, kernel_size=(11, 11), strides=(4, 4), padding='valid', activation='relu'))
# Max Pooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
# 2nd Convolutional Layer
model.add(Conv2D(filters=256, kernel_size=(11, 11), strides=(1, 1), padding='valid'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
# Max Pooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
#dropout
model.add(Dropout(0.25))
# 3rd Convolutional Layer
model.add(Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), padding='valid'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
# 4th Convolutional Layer
model.add(Conv2D(filters=384, kernel_size=(3, 3), strides=(1, 1), padding='valid'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
#dropout
model.add(Dropout(0.25))
# 5th Convolutional Layer
model.add(Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding='valid'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
# Max Pooling
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='valid'))
# Flatten the feature maps to pass them to Fully Connected Layers
model.add(Flatten())
# Fully Connected Layers
model.add(Dense(4096))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dense(4096))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
#dropout
model.add(Dropout(0.5))
model.add(Dense(number_of_classes, activation='softmax'))
model.summary()
return model
def vgg_16_do_bn(input_data_shape=(224, 224, 3), number_of_classes=10):
model = Sequential()
# Block 1
model.add(Conv2D(filters=64, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=64, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Block 2
model.add(Conv2D(filters=128, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=128, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
#dropout
model.add(Dropout(0.25))
# Block 3
model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=256, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Block 4
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
#dropout
model.add(Dropout(0.25))
# Block 5
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=512, input_shape=input_data_shape, kernel_size=(3, 3), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# Flatten the feature maps to pass them to Fully Connected Layers
model.add(Flatten())
# fully connected layers
model.add(Dense(4096))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dense(4096))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
#dropout
model.add(Dropout(0.5))
model.add(Dense(number_of_classes, activation='softmax'))
# Create model.
model.summary()
return model
def custom_model_builder_do_bn(input_data_shape=(224, 224, 3), number_of_classes=10):
model = Sequential()
#Block 1
model.add(Conv2D(filters=32,kernel_size=(5,5),padding='same',input_shape = input_data_shape ))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=32,kernel_size=(5,5),padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding = 'same'))
#Block 2
model.add(Conv2D(filters=64,kernel_size=(5,5),padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=64,kernel_size=(5,5),padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
#dropout
model.add(Dropout(0.25))
#Block 3
model.add(Conv2D(filters=128,kernel_size=(5,5),padding='valid'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=128,kernel_size=(5,5),padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
#Block 4
model.add(Conv2D(filters=256,kernel_size=(5, 5), padding='same'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(filters=256,kernel_size=(5, 5), padding='valid'))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2),padding='same'))
#dropout
model.add(Dropout(0.25))
#flattening the model
model.add(Flatten())
#add dense layers
model.add(Dense(2024))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dense(512))
#adding batch normalization
model.add(BatchNormalization())
model.add(Activation('relu'))
#dropout
model.add(Dropout(0.5))
model.add(Dense(number_of_classes,activation='softmax'))
return model
alexnet_model_do_bn = alexnet_do_bn(input_data_shape=input_data_shape)
vgg16_model_do_bn = vgg_16_do_bn(input_data_shape=input_data_shape)
W0710 02:09:23.779240 139695120390016 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1834: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.
W0710 02:09:23.872189 139695120390016 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_27 (Conv2D) (None, 54, 54, 96) 34944
_________________________________________________________________
max_pooling2d_13 (MaxPooling (None, 27, 27, 96) 0
_________________________________________________________________
conv2d_28 (Conv2D) (None, 17, 17, 256) 2973952
_________________________________________________________________
batch_normalization_1 (Batch (None, 17, 17, 256) 1024
_________________________________________________________________
activation_1 (Activation) (None, 17, 17, 256) 0
_________________________________________________________________
max_pooling2d_14 (MaxPooling (None, 8, 8, 256) 0
_________________________________________________________________
dropout_1 (Dropout) (None, 8, 8, 256) 0
_________________________________________________________________
conv2d_29 (Conv2D) (None, 6, 6, 384) 885120
_________________________________________________________________
batch_normalization_2 (Batch (None, 6, 6, 384) 1536
_________________________________________________________________
activation_2 (Activation) (None, 6, 6, 384) 0
_________________________________________________________________
conv2d_30 (Conv2D) (None, 4, 4, 384) 1327488
_________________________________________________________________
batch_normalization_3 (Batch (None, 4, 4, 384) 1536
_________________________________________________________________
activation_3 (Activation) (None, 4, 4, 384) 0
_________________________________________________________________
dropout_2 (Dropout) (None, 4, 4, 384) 0
_________________________________________________________________
conv2d_31 (Conv2D) (None, 2, 2, 256) 884992
_________________________________________________________________
batch_normalization_4 (Batch (None, 2, 2, 256) 1024
_________________________________________________________________
activation_4 (Activation) (None, 2, 2, 256) 0
_________________________________________________________________
max_pooling2d_15 (MaxPooling (None, 1, 1, 256) 0
_________________________________________________________________
flatten_4 (Flatten) (None, 256) 0
_________________________________________________________________
dense_10 (Dense) (None, 4096) 1052672
_________________________________________________________________
batch_normalization_5 (Batch (None, 4096) 16384
_________________________________________________________________
activation_5 (Activation) (None, 4096) 0
_________________________________________________________________
dense_11 (Dense) (None, 4096) 16781312
_________________________________________________________________
batch_normalization_6 (Batch (None, 4096) 16384
_________________________________________________________________
activation_6 (Activation) (None, 4096) 0
_________________________________________________________________
dropout_3 (Dropout) (None, 4096) 0
_________________________________________________________________
dense_12 (Dense) (None, 10) 40970
=================================================================
Total params: 24,019,338
Trainable params: 24,000,394
Non-trainable params: 18,944
_________________________________________________________________
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_32 (Conv2D) (None, 224, 224, 64) 1792
_________________________________________________________________
batch_normalization_7 (Batch (None, 224, 224, 64) 256
_________________________________________________________________
activation_7 (Activation) (None, 224, 224, 64) 0
_________________________________________________________________
conv2d_33 (Conv2D) (None, 224, 224, 64) 36928
_________________________________________________________________
batch_normalization_8 (Batch (None, 224, 224, 64) 256
_________________________________________________________________
activation_8 (Activation) (None, 224, 224, 64) 0
_________________________________________________________________
max_pooling2d_16 (MaxPooling (None, 112, 112, 64) 0
_________________________________________________________________
conv2d_34 (Conv2D) (None, 112, 112, 128) 73856
_________________________________________________________________
batch_normalization_9 (Batch (None, 112, 112, 128) 512
_________________________________________________________________
activation_9 (Activation) (None, 112, 112, 128) 0
_________________________________________________________________
conv2d_35 (Conv2D) (None, 112, 112, 128) 147584
_________________________________________________________________
batch_normalization_10 (Batc (None, 112, 112, 128) 512
_________________________________________________________________
activation_10 (Activation) (None, 112, 112, 128) 0
_________________________________________________________________
max_pooling2d_17 (MaxPooling (None, 56, 56, 128) 0
_________________________________________________________________
dropout_4 (Dropout) (None, 56, 56, 128) 0
_________________________________________________________________
conv2d_36 (Conv2D) (None, 56, 56, 256) 295168
_________________________________________________________________
batch_normalization_11 (Batc (None, 56, 56, 256) 1024
_________________________________________________________________
activation_11 (Activation) (None, 56, 56, 256) 0
_________________________________________________________________
conv2d_37 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
batch_normalization_12 (Batc (None, 56, 56, 256) 1024
_________________________________________________________________
activation_12 (Activation) (None, 56, 56, 256) 0
_________________________________________________________________
conv2d_38 (Conv2D) (None, 56, 56, 256) 590080
_________________________________________________________________
batch_normalization_13 (Batc (None, 56, 56, 256) 1024
_________________________________________________________________
activation_13 (Activation) (None, 56, 56, 256) 0
_________________________________________________________________
max_pooling2d_18 (MaxPooling (None, 28, 28, 256) 0
_________________________________________________________________
conv2d_39 (Conv2D) (None, 28, 28, 512) 1180160
_________________________________________________________________
batch_normalization_14 (Batc (None, 28, 28, 512) 2048
_________________________________________________________________
activation_14 (Activation) (None, 28, 28, 512) 0
_________________________________________________________________
conv2d_40 (Conv2D) (None, 28, 28, 512) 2359808
_________________________________________________________________
batch_normalization_15 (Batc (None, 28, 28, 512) 2048
_________________________________________________________________
activation_15 (Activation) (None, 28, 28, 512) 0
_________________________________________________________________
conv2d_41 (Conv2D) (None, 28, 28, 512) 2359808
_________________________________________________________________
batch_normalization_16 (Batc (None, 28, 28, 512) 2048
_________________________________________________________________
activation_16 (Activation) (None, 28, 28, 512) 0
_________________________________________________________________
max_pooling2d_19 (MaxPooling (None, 14, 14, 512) 0
_________________________________________________________________
dropout_5 (Dropout) (None, 14, 14, 512) 0
_________________________________________________________________
conv2d_42 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
batch_normalization_17 (Batc (None, 14, 14, 512) 2048
_________________________________________________________________
activation_17 (Activation) (None, 14, 14, 512) 0
_________________________________________________________________
conv2d_43 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
batch_normalization_18 (Batc (None, 14, 14, 512) 2048
_________________________________________________________________
activation_18 (Activation) (None, 14, 14, 512) 0
_________________________________________________________________
conv2d_44 (Conv2D) (None, 14, 14, 512) 2359808
_________________________________________________________________
batch_normalization_19 (Batc (None, 14, 14, 512) 2048
_________________________________________________________________
activation_19 (Activation) (None, 14, 14, 512) 0
_________________________________________________________________
max_pooling2d_20 (MaxPooling (None, 7, 7, 512) 0
_________________________________________________________________
flatten_5 (Flatten) (None, 25088) 0
_________________________________________________________________
dense_13 (Dense) (None, 4096) 102764544
_________________________________________________________________
batch_normalization_20 (Batc (None, 4096) 16384
_________________________________________________________________
activation_20 (Activation) (None, 4096) 0
_________________________________________________________________
dense_14 (Dense) (None, 4096) 16781312
_________________________________________________________________
batch_normalization_21 (Batc (None, 4096) 16384
_________________________________________________________________
activation_21 (Activation) (None, 4096) 0
_________________________________________________________________
dropout_6 (Dropout) (None, 4096) 0
_________________________________________________________________
dense_15 (Dense) (None, 10) 40970
=================================================================
Total params: 134,351,178
Trainable params: 134,326,346
Non-trainable params: 24,832
_________________________________________________________________
custom_model_do_bn = custom_model_builder_do_bn(input_data_shape=input_data_shape)
custom_model_do_bn.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_45 (Conv2D) (None, 224, 224, 32) 2432
_________________________________________________________________
batch_normalization_22 (Batc (None, 224, 224, 32) 128
_________________________________________________________________
activation_22 (Activation) (None, 224, 224, 32) 0
_________________________________________________________________
conv2d_46 (Conv2D) (None, 224, 224, 32) 25632
_________________________________________________________________
batch_normalization_23 (Batc (None, 224, 224, 32) 128
_________________________________________________________________
activation_23 (Activation) (None, 224, 224, 32) 0
_________________________________________________________________
max_pooling2d_21 (MaxPooling (None, 112, 112, 32) 0
_________________________________________________________________
conv2d_47 (Conv2D) (None, 112, 112, 64) 51264
_________________________________________________________________
batch_normalization_24 (Batc (None, 112, 112, 64) 256
_________________________________________________________________
activation_24 (Activation) (None, 112, 112, 64) 0
_________________________________________________________________
conv2d_48 (Conv2D) (None, 112, 112, 64) 102464
_________________________________________________________________
batch_normalization_25 (Batc (None, 112, 112, 64) 256
_________________________________________________________________
activation_25 (Activation) (None, 112, 112, 64) 0
_________________________________________________________________
max_pooling2d_22 (MaxPooling (None, 56, 56, 64) 0
_________________________________________________________________
dropout_7 (Dropout) (None, 56, 56, 64) 0
_________________________________________________________________
conv2d_49 (Conv2D) (None, 52, 52, 128) 204928
_________________________________________________________________
batch_normalization_26 (Batc (None, 52, 52, 128) 512
_________________________________________________________________
activation_26 (Activation) (None, 52, 52, 128) 0
_________________________________________________________________
conv2d_50 (Conv2D) (None, 52, 52, 128) 409728
_________________________________________________________________
batch_normalization_27 (Batc (None, 52, 52, 128) 512
_________________________________________________________________
activation_27 (Activation) (None, 52, 52, 128) 0
_________________________________________________________________
max_pooling2d_23 (MaxPooling (None, 26, 26, 128) 0
_________________________________________________________________
conv2d_51 (Conv2D) (None, 26, 26, 256) 819456
_________________________________________________________________
batch_normalization_28 (Batc (None, 26, 26, 256) 1024
_________________________________________________________________
activation_28 (Activation) (None, 26, 26, 256) 0
_________________________________________________________________
conv2d_52 (Conv2D) (None, 22, 22, 256) 1638656
_________________________________________________________________
batch_normalization_29 (Batc (None, 22, 22, 256) 1024
_________________________________________________________________
activation_29 (Activation) (None, 22, 22, 256) 0
_________________________________________________________________
max_pooling2d_24 (MaxPooling (None, 11, 11, 256) 0
_________________________________________________________________
dropout_8 (Dropout) (None, 11, 11, 256) 0
_________________________________________________________________
flatten_6 (Flatten) (None, 30976) 0
_________________________________________________________________
dense_16 (Dense) (None, 2024) 62697448
_________________________________________________________________
batch_normalization_30 (Batc (None, 2024) 8096
_________________________________________________________________
activation_30 (Activation) (None, 2024) 0
_________________________________________________________________
dense_17 (Dense) (None, 512) 1036800
_________________________________________________________________
batch_normalization_31 (Batc (None, 512) 2048
_________________________________________________________________
activation_31 (Activation) (None, 512) 0
_________________________________________________________________
dropout_9 (Dropout) (None, 512) 0
_________________________________________________________________
dense_18 (Dense) (None, 10) 5130
=================================================================
Total params: 67,007,922
Trainable params: 67,000,930
Non-trainable params: 6,992
_________________________________________________________________
custom_model_do_bn = custom_model_builder_do_bn(input_data_shape=input_data_shape)
custom_model_do_bn.summary()
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
conv2d_53 (Conv2D) (None, 224, 224, 32) 2432
_________________________________________________________________
batch_normalization_32 (Batc (None, 224, 224, 32) 128
_________________________________________________________________
activation_32 (Activation) (None, 224, 224, 32) 0
_________________________________________________________________
conv2d_54 (Conv2D) (None, 224, 224, 32) 25632
_________________________________________________________________
batch_normalization_33 (Batc (None, 224, 224, 32) 128
_________________________________________________________________
activation_33 (Activation) (None, 224, 224, 32) 0
_________________________________________________________________
max_pooling2d_25 (MaxPooling (None, 112, 112, 32) 0
_________________________________________________________________
conv2d_55 (Conv2D) (None, 112, 112, 64) 51264
_________________________________________________________________
batch_normalization_34 (Batc (None, 112, 112, 64) 256
_________________________________________________________________
activation_34 (Activation) (None, 112, 112, 64) 0
_________________________________________________________________
conv2d_56 (Conv2D) (None, 112, 112, 64) 102464
_________________________________________________________________
batch_normalization_35 (Batc (None, 112, 112, 64) 256
_________________________________________________________________
activation_35 (Activation) (None, 112, 112, 64) 0
_________________________________________________________________
max_pooling2d_26 (MaxPooling (None, 56, 56, 64) 0
_________________________________________________________________
dropout_10 (Dropout) (None, 56, 56, 64) 0
_________________________________________________________________
conv2d_57 (Conv2D) (None, 52, 52, 128) 204928
_________________________________________________________________
batch_normalization_36 (Batc (None, 52, 52, 128) 512
_________________________________________________________________
activation_36 (Activation) (None, 52, 52, 128) 0
_________________________________________________________________
conv2d_58 (Conv2D) (None, 52, 52, 128) 409728
_________________________________________________________________
batch_normalization_37 (Batc (None, 52, 52, 128) 512
_________________________________________________________________
activation_37 (Activation) (None, 52, 52, 128) 0
_________________________________________________________________
max_pooling2d_27 (MaxPooling (None, 26, 26, 128) 0
_________________________________________________________________
conv2d_59 (Conv2D) (None, 26, 26, 256) 819456
_________________________________________________________________
batch_normalization_38 (Batc (None, 26, 26, 256) 1024
_________________________________________________________________
activation_38 (Activation) (None, 26, 26, 256) 0
_________________________________________________________________
conv2d_60 (Conv2D) (None, 22, 22, 256) 1638656
_________________________________________________________________
batch_normalization_39 (Batc (None, 22, 22, 256) 1024
_________________________________________________________________
activation_39 (Activation) (None, 22, 22, 256) 0
_________________________________________________________________
max_pooling2d_28 (MaxPooling (None, 11, 11, 256) 0
_________________________________________________________________
dropout_11 (Dropout) (None, 11, 11, 256) 0
_________________________________________________________________
flatten_7 (Flatten) (None, 30976) 0
_________________________________________________________________
dense_19 (Dense) (None, 2024) 62697448
_________________________________________________________________
batch_normalization_40 (Batc (None, 2024) 8096
_________________________________________________________________
activation_40 (Activation) (None, 2024) 0
_________________________________________________________________
dense_20 (Dense) (None, 512) 1036800
_________________________________________________________________
batch_normalization_41 (Batc (None, 512) 2048
_________________________________________________________________
activation_41 (Activation) (None, 512) 0
_________________________________________________________________
dropout_12 (Dropout) (None, 512) 0
_________________________________________________________________
dense_21 (Dense) (None, 10) 5130
=================================================================
Total params: 67,007,922
Trainable params: 67,000,930
Non-trainable params: 6,992
_________________________________________________________________
alexnet_model_do_bn.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
vgg16_model_do_bn.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
custom_model_do_bn.compile(loss=keras.losses.categorical_crossentropy, optimizer='adam', metrics=["accuracy"])
alexnet_inference_time_do_bn = []
vgg16_inference_time_do_bn = []
custom_inference_time_do_bn = []
# dummy tensor to check the inference time of each network
x_test = np.random.rand(batch_size, input_data_shape[0], input_data_shape[1], input_data_shape[2])
for _ in range(num_of_training_iteration):
alexnet_inference_start = time.time()
alexnet_inference = alexnet_model_do_bn.predict_classes(x_test)
alexnet_inference_finish = time.time()
alexnet_inference_time_do_bn.append(alexnet_inference_finish - alexnet_inference_start)
print("Average Inference time for AlexNet: {}".format(np.mean(alexnet_inference_time_do_bn)))
for _ in range(num_of_training_iteration):
vgg16_inference_start = time.time()
vgg_inference = vgg16_model_do_bn.predict_classes(x_test)
vgg16_inference_finish = time.time()
vgg16_inference_time_do_bn.append(vgg16_inference_finish - vgg16_inference_start)
print("Average Inference time for VGG-16: {}".format(np.mean(vgg16_inference_time)))
for _ in range(num_of_training_iteration):
custom_inference_start = time.time()
custom_inference = custom_model_do_bn.predict_classes(x_test)
custom_inference_finish = time.time()
custom_inference_time_do_bn.append(custom_inference_finish - custom_inference_start)
print("Average Inference time for Custom Model: {}".format(np.mean(custom_inference_time)))
Average Inference time for AlexNet: 0.204864182472229
Average Inference time for VGG-16: 1.2100702261924743
Average Inference time for Custom Model: 0.5777664661407471
Average Inference time doesn't change much after adding batch normalization and dropout, because in inference time when we are doing batch normalization, we are just adding and multiplying outputs by scalers. In AlexNet inference time increased little bit because it has less parameters, but for other models the batch normalization parameters are very less, compared to other model parameters.**
jovian.commit()
[jovian] Saving notebook..