Jovian
⭐️
Sign In
In [1]:
!pip install -q six numpy scipy matplotlib scikit-image opencv-python imageio
!pip install -q keras imgaug
!pip install -q tensorboardcolab
!pip install -q git+https://www.github.com/keras-team/keras-contrib.git
Building wheel for keras-contrib (setup.py) ... done
In [2]:
import os,sys,math
from matplotlib import pyplot as plt
%matplotlib inline
import numpy as np
import pandas as pd

import keras
from keras.models import Model, Sequential, load_model
from keras.layers import Dropout, Flatten, GlobalAveragePooling2D, AveragePooling2D, merge, Activation, Conv2D, MaxPooling2D, Input
from keras.layers import BatchNormalization, LeakyReLU, SeparableConv2D, Lambda, Concatenate,Softmax
from keras.optimizers import SGD, Adam
from keras.preprocessing.image import ImageDataGenerator
from keras.utils import np_utils
from keras.callbacks import ModelCheckpoint, LearningRateScheduler, ReduceLROnPlateau, CSVLogger, EarlyStopping
Using TensorFlow backend.
In [ ]:
from keras_contrib.callbacks import CyclicLR
import imgaug as ia
from imgaug import augmenters as iaa
from tensorboardcolab import TensorBoardColab, TensorBoardColabCallback
In [ ]:
# this part will prevent tensorflow to allocate all the avaliable GPU Memory
# backend
import tensorflow as tf
from keras import backend as k

# Don't pre-allocate memory; allocate as-needed
config = tf.ConfigProto()
config.gpu_options.allow_growth = True

# Create a session with the above options specified.
k.tensorflow_backend.set_session(tf.Session(config=config))
In [5]:
import google
colab_dir='./'
weights_dir='./'
file_name='EIP_CIFAR_10'
if hasattr(google,'colab'):
    from google.colab import drive
    drive.mount('/content/gdrive')
    colab_dir='/content/gdrive/My Drive/Colab Notebooks/'
    weights_dir=colab_dir+'weights/'
model_file=colab_dir+file_name+'.h5'
In [ ]:
def create_dataset(image_dim=64,augment=True,batch_size=256):
  file_exists = os.path.exists("tiny-imagenet-200.zip")
  if file_exists:
    print("Files present. Skipping.")
  else:
    print(os.popen('wget http://cs231n.stanford.edu/tiny-imagenet-200.zip').read())
    print(os.popen('unzip -qq tiny-imagenet-200.zip').read())
    
  val_data = pd.read_csv('./tiny-imagenet-200/val/val_annotations.txt', sep='\t', header=None, names=['File', 'Class', 'X', 'Y', 'H', 'W'])
  val_data.drop(['X', 'Y', 'H', 'W'], axis=1, inplace=True)
  train_datagen = None
  
  if augment:
    train_datagen = ImageDataGenerator(rescale= 1./255,fill_mode='nearest',
      zoom_range = 0.2,horizontal_flip=True,shear_range=0.2,
      rotation_range=40,width_shift_range=0.2,height_shift_range=0.2,
      featurewise_std_normalization=True)
  else:
    train_datagen = ImageDataGenerator(rescale= 1./255,featurewise_std_normalization=True)
  
  valid_datagen = ImageDataGenerator(rescale=1./255)
  train_generator = train_datagen.flow_from_directory( r'./tiny-imagenet-200/train/', target_size=(image_dim, image_dim), color_mode='rgb',
                                                    batch_size=batch_size, class_mode='categorical', shuffle=True, interpolation='bicubic')
  
  validation_generator = valid_datagen.flow_from_dataframe(val_data, directory='./tiny-imagenet-200/val/images/', x_col='File', y_col='Class', target_size=(image_dim, image_dim),
                                                    color_mode='rgb', class_mode='categorical', batch_size=batch_size, shuffle=True,interpolation='bicubic')
  #x_batch, y_val_batch = next(train_generator)
  #print('Training batch shape:',x_batch.shape)
  
  return train_generator, validation_generator#, x_batch, y_val_batch
In [ ]:
#Augmentation and resizing
#augment and then concat samples with original
def augment(dataset,flip=0.5,blur=1.0):
    ia.seed(1)
    seq = iaa.Sequential([iaa.Crop(px=(0, 16)),iaa.Fliplr(0.5),iaa.GaussianBlur(sigma=(0, 3.0))], random_order=True)   
    return seq.augment_images(dataset)

def augmenter(X,y,start=0,end=1,append=False):
    ln=len(X)
    print('Before augmentation:',X.shape,y.shape)
    start=int(start*ln)
    end=int(end*ln)
    if append:
      new_X=augment(X)[start:end]
      new_y=y[start:end]
      X=np.concatenate((X,new_X))
      y=np.concatenate((y,new_y))
    else:
      X=augment(X)
    print('After augmentation:',X.shape,y.shape)
    return (X,y)

#26x26 is almost half of 32x32. 22x22 maybe too small even though its exact half.
def resize_imgs(imgs,shape=(26,26)):
    seq = iaa.Sequential([iaa.Scale({"height": shape[0], "width": shape[1]})])
    return seq.augment_images(imgs)
In [ ]:
modval=50
max_lr=0.1

def cosine_anneal_schedule(t,lr):
  global modval,max_lr
  modval=int(modval)
  cos_inner = (np.pi * (t % modval))/ modval
  cos_out = np.cos(cos_inner) + 1
  return float(max_lr / 2 * cos_out)

def load_prev_model(model_small=None):
#     print(os.popen('du -sh '+colab_dir+'*').read())
    last_best=os.popen('du -sh '+colab_dir+r'*|sed -r "s/^[0-9\.]+[MK]?\s(.*)$/\1/g;"|tail -1').read().strip()
    model_prev=None
    try:
        print('Attempting to load last best model from file - ',end='')
        last_best_fn=last_best.split('/')[-1]
        last_best_epoch=last_best_fn.split('.')[1].split('-')[0]
        model_prev=load_model(last_best)
        print('Sucess\nLoaded '+last_best_fn, 'epoch :', last_best_epoch)
    except Exception as e:
        print('Failed!\n',e)
        try:
            print('Attempting to load last saved model from file - ',end='')
            model_prev = load_model(model_file)
            print('Sucess\nLoaded model from file',model_file)
        except Exception as e:
    #         print(str(e), 'at line ', sys.exc_info()[2].tb_lineno)
            print('Failed!\n',e)
            try:
                print('Attempting to load in memory, small model - ',end='')
                if len(model_small.layers)>1:
                    model_prev=model_small
                print('Sucess')
            except Exception as e:
                print('Failed!\n',e)
    return model_prev
  
def copy_weights(model_to,model_from):
    #model_to.set_weights(model_from.get_weights())
    s,err=0,0
    print('Trying to copy weights')
    try:
        if len(model_to.layers) >1 and len(model_from.layers) >1:
            pass
    except Exception as e:
        print('Inavlid models',model_to,model_from)
        return
    for new_layer, layer in zip(model_to.layers[1:], model_from.layers[1:]):
        s+=1
        try:
            new_layer.set_weights(layer.get_weights())
        except Exception as e:
            pass
    print('Done: errors:',err)
      
#new model with larger i/p layer
def larger_model(src_model,shape=(32,32,3)):
    new_input=Input(shape)
    src_model.layers.pop(0)
    new_output=src_model(new_input)
    new_model=Model(new_input,new_output)
    return new_model

# Load CIFAR10 Data
def load_data(resize=False,append=False,shape=(26,26),train_augment=True,test_augment=False):
    (x_train, y_train), (x_test, y_test) = cifar10.load_data()
    
    y_train = keras.utils.to_categorical(y_train, num_classes)
    y_test = keras.utils.to_categorical(y_test, num_classes)
    
    if resize:
        x_train=resize_imgs(x_train,shape)
        x_test=resize_imgs(x_test,shape)
    if train_augment:
      (x_train, y_train) = augmenter(x_train, y_train,end=1,append=append)
    if test_augment:
        (x_test, y_test) = augmenter(x_test, y_test,end=1,append=append)
    return (x_train, y_train,x_test, y_test)
#create a dnn model
def create_model(input_shape,num_layers,input_conv_filters=12,input=None):
    print('Creating model with input shape',input_shape)
    if input is None:
        input = Input(input_shape)
    First_Conv2D = Conv2D(input_conv_filters, (3,3), use_bias=False ,padding='same')(input)
    hidden_dense_blocks = dense_units_chain(n_dense_blocks,First_Conv2D,num_filter,dropout_rate,num_layers)
    Last_Block = add_denseblock(hidden_dense_blocks, num_filter, dropout_rate)
    output = output_layer(Last_Block)
    model = Model(inputs=[input], outputs=[output])
    return model
In [ ]:
sc=False
after_transistion = True
# Dense Block
def add_denseblock(input,num_layers, growth_rate = 12, dropout_rate = 0.2, bn_layers=48, middle_layers=24):
    
    temp = input
    
    #if len(Block_outputs)>1:
    #  bv = Block_outputs[-2]
    #  #temp = Concatenate(axis=-1)([Block_outputs[-2],[temp]])
    #  print(bv)
    
    for _ in range(num_layers):
        ######################################################################################################################################
        BatchNorm = BatchNormalization()(temp)
        relu = Activation('relu')(BatchNorm)        
        Conv2D_1_1_bottleneck = Conv2D(bn_layers, (1,1), use_bias=False ,padding='same',kernel_initializer='he_normal')(relu) #bottleneck  <===
        ###################################################################################################################################### |
        #BatchNorm = BatchNormalization()(Conv2D_1_1_bottleneck)
        #relu = Activation('relu')(BatchNorm)        
        #Conv2D_1_1_bottleneck = Conv2D(bn_layers, (3,3), use_bias=False ,padding='same',kernel_initializer='he_normal')(relu) #bottleneck  <===
        ###################################################################################################################################### |
        #BatchNorm = BatchNormalization()(Conv2D_1_1_bottleneck) #extra layer                                                                 # |
        #relu = Activation('relu')(BatchNorm) #extra layer                                                                                    # |
        #Conv2D_1_1 = Conv2D(middle_layers, (1,1), use_bias=False ,padding='same',kernel_initializer='he_normal')(relu) #extra layer          # |
        ###################################################################################################################################### |
        BatchNorm = BatchNormalization()(Conv2D_1_1_bottleneck) #bottleneck   <================================================================
        relu = Activation('relu')(BatchNorm)
        Conv2D_3_3 = Conv2D(growth_rate, (3,3), use_bias=False ,padding='same',kernel_initializer='he_normal')(relu)
        ######################################################################################################################################|
        
        if dropout_rate>0:
          Conv2D_3_3 = Dropout(dropout_rate)(Conv2D_3_3)
        concat = Concatenate(axis=-1)([temp,Conv2D_3_3])
        
        temp = concat
    #if not after_transistion:
    #  Block_outputs.append(temp)
    return temp

def add_transition(input, num_filter = 12, dropout_rate = 0.2):
    BatchNorm = BatchNormalization()(input)
    relu = Activation('relu')(BatchNorm)
    Conv2D_BottleNeck = Conv2D(int(num_filter), (1,1), use_bias=False ,padding='same')(relu)
    if dropout_rate>0:
      Conv2D_BottleNeck = Dropout(dropout_rate)(Conv2D_BottleNeck)
    avg = AveragePooling2D(pool_size=(2,2),padding='same')(Conv2D_BottleNeck)
    #if after_transistion:
    #  Block_outputs.append(avg)
    return avg

  
def output_layer(input,num_classes,mf=2,nl=2):
    inp = input
    for i in range(nl,-1,-1):
      classes = int(num_classes*(mf**i))
      BatchNorm = BatchNormalization()(inp)
      relu = Activation('relu')(BatchNorm)
      conv1_1 = Conv2D(classes,(1,1),use_bias=False,kernel_initializer='he_normal',padding='same')(relu)
      conv3_3 = Conv2D(classes,(3,3),use_bias=False,kernel_initializer='he_normal',padding='same')(conv1_1)
      inp = conv3_3
    
    GlobalAveragePooling = GlobalAveragePooling2D(data_format='channels_last')(inp)
    sf = Softmax()(GlobalAveragePooling)
    return sf
In [ ]:
def custom_model(input_shape,initial_layers,num_layers,db_arch,dropout_rate=0,trans_2k=False,n_blocks=3,omf=2,onl=2):
  Block,Transition = None,None
  bn_layers,middle_layers,growth_rate = db_arch
  
  input = Input(shape=input_shape)
  First_Conv2D = Conv2D(initial_layers, (3,3), use_bias=False ,padding='same')(input)
  num_filter=initial_layers
  
  input_layer = First_Conv2D
  for i in range(n_blocks):
    Block = add_denseblock(input_layer,num_layers, growth_rate, dropout_rate,bn_layers,middle_layers)
    
    num_filter+=growth_rate*num_layers
    num_filter=(growth_rate*2) if trans_2k else num_filter//2    
    
    Transition = add_transition(Block, num_filter, dropout_rate)    
    input_layer = Transition
  
  #Last_Block = add_denseblock(Transition, num_layers, growth_rate, dropout_rate,bn_layers,middle_layers)
  output = output_layer(Block,num_classes,omf,onl)
  model = Model(inputs=[input], outputs=[output])
  return model
In [11]:
print('====================HYPER PARAMETERS====================')
====================HYPER PARAMETERS====================
In [ ]:
num_filter = 0 # add it up as you go
compression = 0.5
dropout_rate =0#0.2#.5
num_classes = 200
train_ds_size=100000
validation_ds_size=10000

modval=20
max_lr=0.3
image_dim = 32
batch_size = 64
inp_shape = (image_dim,image_dim,3)

n_blocks=3
num_layers=10#26
initial_layers=24
db_archs=[[128,64,32],[96,48,24],[48,24,12]]
dbi = 1
db_arch=db_archs[dbi]

hist=[]
epochs=initial_epoch=0
Block_outputs = []
In [13]:
model = custom_model(inp_shape,initial_layers,num_layers,db_arch,n_blocks=n_blocks,trans_2k=False,omf=2,onl=0)
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version. Instructions for updating: Colocations handled automatically by placer.
In [14]:
tbc=TensorBoardColabCallback(TensorBoardColab())
hist=[]
Wait for 8 seconds... TensorBoard link: https://55ec2c29.ngrok.io
In [15]:
model.compile(loss='categorical_crossentropy',metrics=['accuracy'],optimizer=SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True))
model.summary()
__________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ================================================================================================== input_1 (InputLayer) (None, 32, 32, 3) 0 __________________________________________________________________________________________________ conv2d_1 (Conv2D) (None, 32, 32, 24) 648 input_1[0][0] __________________________________________________________________________________________________ batch_normalization_1 (BatchNor (None, 32, 32, 24) 96 conv2d_1[0][0] __________________________________________________________________________________________________ activation_1 (Activation) (None, 32, 32, 24) 0 batch_normalization_1[0][0] __________________________________________________________________________________________________ conv2d_2 (Conv2D) (None, 32, 32, 96) 2304 activation_1[0][0] __________________________________________________________________________________________________ batch_normalization_2 (BatchNor (None, 32, 32, 96) 384 conv2d_2[0][0] __________________________________________________________________________________________________ activation_2 (Activation) (None, 32, 32, 96) 0 batch_normalization_2[0][0] __________________________________________________________________________________________________ conv2d_3 (Conv2D) (None, 32, 32, 24) 20736 activation_2[0][0] __________________________________________________________________________________________________ concatenate_1 (Concatenate) (None, 32, 32, 48) 0 conv2d_1[0][0] conv2d_3[0][0] __________________________________________________________________________________________________ batch_normalization_3 (BatchNor (None, 32, 32, 48) 192 concatenate_1[0][0] __________________________________________________________________________________________________ activation_3 (Activation) (None, 32, 32, 48) 0 batch_normalization_3[0][0] __________________________________________________________________________________________________ conv2d_4 (Conv2D) (None, 32, 32, 96) 4608 activation_3[0][0] __________________________________________________________________________________________________ batch_normalization_4 (BatchNor (None, 32, 32, 96) 384 conv2d_4[0][0] __________________________________________________________________________________________________ activation_4 (Activation) (None, 32, 32, 96) 0 batch_normalization_4[0][0] __________________________________________________________________________________________________ conv2d_5 (Conv2D) (None, 32, 32, 24) 20736 activation_4[0][0] __________________________________________________________________________________________________ concatenate_2 (Concatenate) (None, 32, 32, 72) 0 concatenate_1[0][0] conv2d_5[0][0] __________________________________________________________________________________________________ batch_normalization_5 (BatchNor (None, 32, 32, 72) 288 concatenate_2[0][0] __________________________________________________________________________________________________ activation_5 (Activation) (None, 32, 32, 72) 0 batch_normalization_5[0][0] __________________________________________________________________________________________________ conv2d_6 (Conv2D) (None, 32, 32, 96) 6912 activation_5[0][0] __________________________________________________________________________________________________ batch_normalization_6 (BatchNor (None, 32, 32, 96) 384 conv2d_6[0][0] __________________________________________________________________________________________________ activation_6 (Activation) (None, 32, 32, 96) 0 batch_normalization_6[0][0] __________________________________________________________________________________________________ conv2d_7 (Conv2D) (None, 32, 32, 24) 20736 activation_6[0][0] __________________________________________________________________________________________________ concatenate_3 (Concatenate) (None, 32, 32, 96) 0 concatenate_2[0][0] conv2d_7[0][0] __________________________________________________________________________________________________ batch_normalization_7 (BatchNor (None, 32, 32, 96) 384 concatenate_3[0][0] __________________________________________________________________________________________________ activation_7 (Activation) (None, 32, 32, 96) 0 batch_normalization_7[0][0] __________________________________________________________________________________________________ conv2d_8 (Conv2D) (None, 32, 32, 96) 9216 activation_7[0][0] __________________________________________________________________________________________________ batch_normalization_8 (BatchNor (None, 32, 32, 96) 384 conv2d_8[0][0] __________________________________________________________________________________________________ activation_8 (Activation) (None, 32, 32, 96) 0 batch_normalization_8[0][0] __________________________________________________________________________________________________ conv2d_9 (Conv2D) (None, 32, 32, 24) 20736 activation_8[0][0] __________________________________________________________________________________________________ concatenate_4 (Concatenate) (None, 32, 32, 120) 0 concatenate_3[0][0] conv2d_9[0][0] __________________________________________________________________________________________________ batch_normalization_9 (BatchNor (None, 32, 32, 120) 480 concatenate_4[0][0] __________________________________________________________________________________________________ activation_9 (Activation) (None, 32, 32, 120) 0 batch_normalization_9[0][0] __________________________________________________________________________________________________ conv2d_10 (Conv2D) (None, 32, 32, 96) 11520 activation_9[0][0] __________________________________________________________________________________________________ batch_normalization_10 (BatchNo (None, 32, 32, 96) 384 conv2d_10[0][0] __________________________________________________________________________________________________ activation_10 (Activation) (None, 32, 32, 96) 0 batch_normalization_10[0][0] __________________________________________________________________________________________________ conv2d_11 (Conv2D) (None, 32, 32, 24) 20736 activation_10[0][0] __________________________________________________________________________________________________ concatenate_5 (Concatenate) (None, 32, 32, 144) 0 concatenate_4[0][0] conv2d_11[0][0] __________________________________________________________________________________________________ batch_normalization_11 (BatchNo (None, 32, 32, 144) 576 concatenate_5[0][0] __________________________________________________________________________________________________ activation_11 (Activation) (None, 32, 32, 144) 0 batch_normalization_11[0][0] __________________________________________________________________________________________________ conv2d_12 (Conv2D) (None, 32, 32, 96) 13824 activation_11[0][0] __________________________________________________________________________________________________ batch_normalization_12 (BatchNo (None, 32, 32, 96) 384 conv2d_12[0][0] __________________________________________________________________________________________________ activation_12 (Activation) (None, 32, 32, 96) 0 batch_normalization_12[0][0] __________________________________________________________________________________________________ conv2d_13 (Conv2D) (None, 32, 32, 24) 20736 activation_12[0][0] __________________________________________________________________________________________________ concatenate_6 (Concatenate) (None, 32, 32, 168) 0 concatenate_5[0][0] conv2d_13[0][0] __________________________________________________________________________________________________ batch_normalization_13 (BatchNo (None, 32, 32, 168) 672 concatenate_6[0][0] __________________________________________________________________________________________________ activation_13 (Activation) (None, 32, 32, 168) 0 batch_normalization_13[0][0] __________________________________________________________________________________________________ conv2d_14 (Conv2D) (None, 32, 32, 96) 16128 activation_13[0][0] __________________________________________________________________________________________________ batch_normalization_14 (BatchNo (None, 32, 32, 96) 384 conv2d_14[0][0] __________________________________________________________________________________________________ activation_14 (Activation) (None, 32, 32, 96) 0 batch_normalization_14[0][0] __________________________________________________________________________________________________ conv2d_15 (Conv2D) (None, 32, 32, 24) 20736 activation_14[0][0] __________________________________________________________________________________________________ concatenate_7 (Concatenate) (None, 32, 32, 192) 0 concatenate_6[0][0] conv2d_15[0][0] __________________________________________________________________________________________________ batch_normalization_15 (BatchNo (None, 32, 32, 192) 768 concatenate_7[0][0] __________________________________________________________________________________________________ activation_15 (Activation) (None, 32, 32, 192) 0 batch_normalization_15[0][0] __________________________________________________________________________________________________ conv2d_16 (Conv2D) (None, 32, 32, 96) 18432 activation_15[0][0] __________________________________________________________________________________________________ batch_normalization_16 (BatchNo (None, 32, 32, 96) 384 conv2d_16[0][0] __________________________________________________________________________________________________ activation_16 (Activation) (None, 32, 32, 96) 0 batch_normalization_16[0][0] __________________________________________________________________________________________________ conv2d_17 (Conv2D) (None, 32, 32, 24) 20736 activation_16[0][0] __________________________________________________________________________________________________ concatenate_8 (Concatenate) (None, 32, 32, 216) 0 concatenate_7[0][0] conv2d_17[0][0] __________________________________________________________________________________________________ batch_normalization_17 (BatchNo (None, 32, 32, 216) 864 concatenate_8[0][0] __________________________________________________________________________________________________ activation_17 (Activation) (None, 32, 32, 216) 0 batch_normalization_17[0][0] __________________________________________________________________________________________________ conv2d_18 (Conv2D) (None, 32, 32, 96) 20736 activation_17[0][0] __________________________________________________________________________________________________ batch_normalization_18 (BatchNo (None, 32, 32, 96) 384 conv2d_18[0][0] __________________________________________________________________________________________________ activation_18 (Activation) (None, 32, 32, 96) 0 batch_normalization_18[0][0] __________________________________________________________________________________________________ conv2d_19 (Conv2D) (None, 32, 32, 24) 20736 activation_18[0][0] __________________________________________________________________________________________________ concatenate_9 (Concatenate) (None, 32, 32, 240) 0 concatenate_8[0][0] conv2d_19[0][0] __________________________________________________________________________________________________ batch_normalization_19 (BatchNo (None, 32, 32, 240) 960 concatenate_9[0][0] __________________________________________________________________________________________________ activation_19 (Activation) (None, 32, 32, 240) 0 batch_normalization_19[0][0] __________________________________________________________________________________________________ conv2d_20 (Conv2D) (None, 32, 32, 96) 23040 activation_19[0][0] __________________________________________________________________________________________________ batch_normalization_20 (BatchNo (None, 32, 32, 96) 384 conv2d_20[0][0] __________________________________________________________________________________________________ activation_20 (Activation) (None, 32, 32, 96) 0 batch_normalization_20[0][0] __________________________________________________________________________________________________ conv2d_21 (Conv2D) (None, 32, 32, 24) 20736 activation_20[0][0] __________________________________________________________________________________________________ concatenate_10 (Concatenate) (None, 32, 32, 264) 0 concatenate_9[0][0] conv2d_21[0][0] __________________________________________________________________________________________________ batch_normalization_21 (BatchNo (None, 32, 32, 264) 1056 concatenate_10[0][0] __________________________________________________________________________________________________ activation_21 (Activation) (None, 32, 32, 264) 0 batch_normalization_21[0][0] __________________________________________________________________________________________________ conv2d_22 (Conv2D) (None, 32, 32, 132) 34848 activation_21[0][0] __________________________________________________________________________________________________ average_pooling2d_1 (AveragePoo (None, 16, 16, 132) 0 conv2d_22[0][0] __________________________________________________________________________________________________ batch_normalization_22 (BatchNo (None, 16, 16, 132) 528 average_pooling2d_1[0][0] __________________________________________________________________________________________________ activation_22 (Activation) (None, 16, 16, 132) 0 batch_normalization_22[0][0] __________________________________________________________________________________________________ conv2d_23 (Conv2D) (None, 16, 16, 96) 12672 activation_22[0][0] __________________________________________________________________________________________________ batch_normalization_23 (BatchNo (None, 16, 16, 96) 384 conv2d_23[0][0] __________________________________________________________________________________________________ activation_23 (Activation) (None, 16, 16, 96) 0 batch_normalization_23[0][0] __________________________________________________________________________________________________ conv2d_24 (Conv2D) (None, 16, 16, 24) 20736 activation_23[0][0] __________________________________________________________________________________________________ concatenate_11 (Concatenate) (None, 16, 16, 156) 0 average_pooling2d_1[0][0] conv2d_24[0][0] __________________________________________________________________________________________________ batch_normalization_24 (BatchNo (None, 16, 16, 156) 624 concatenate_11[0][0] __________________________________________________________________________________________________ activation_24 (Activation) (None, 16, 16, 156) 0 batch_normalization_24[0][0] __________________________________________________________________________________________________ conv2d_25 (Conv2D) (None, 16, 16, 96) 14976 activation_24[0][0] __________________________________________________________________________________________________ batch_normalization_25 (BatchNo (None, 16, 16, 96) 384 conv2d_25[0][0] __________________________________________________________________________________________________ activation_25 (Activation) (None, 16, 16, 96) 0 batch_normalization_25[0][0] __________________________________________________________________________________________________ conv2d_26 (Conv2D) (None, 16, 16, 24) 20736 activation_25[0][0] __________________________________________________________________________________________________ concatenate_12 (Concatenate) (None, 16, 16, 180) 0 concatenate_11[0][0] conv2d_26[0][0] __________________________________________________________________________________________________ batch_normalization_26 (BatchNo (None, 16, 16, 180) 720 concatenate_12[0][0] __________________________________________________________________________________________________ activation_26 (Activation) (None, 16, 16, 180) 0 batch_normalization_26[0][0] __________________________________________________________________________________________________ conv2d_27 (Conv2D) (None, 16, 16, 96) 17280 activation_26[0][0] __________________________________________________________________________________________________ batch_normalization_27 (BatchNo (None, 16, 16, 96) 384 conv2d_27[0][0] __________________________________________________________________________________________________ activation_27 (Activation) (None, 16, 16, 96) 0 batch_normalization_27[0][0] __________________________________________________________________________________________________ conv2d_28 (Conv2D) (None, 16, 16, 24) 20736 activation_27[0][0] __________________________________________________________________________________________________ concatenate_13 (Concatenate) (None, 16, 16, 204) 0 concatenate_12[0][0] conv2d_28[0][0] __________________________________________________________________________________________________ batch_normalization_28 (BatchNo (None, 16, 16, 204) 816 concatenate_13[0][0] __________________________________________________________________________________________________ activation_28 (Activation) (None, 16, 16, 204) 0 batch_normalization_28[0][0] __________________________________________________________________________________________________ conv2d_29 (Conv2D) (None, 16, 16, 96) 19584 activation_28[0][0] __________________________________________________________________________________________________ batch_normalization_29 (BatchNo (None, 16, 16, 96) 384 conv2d_29[0][0] __________________________________________________________________________________________________ activation_29 (Activation) (None, 16, 16, 96) 0 batch_normalization_29[0][0] __________________________________________________________________________________________________ conv2d_30 (Conv2D) (None, 16, 16, 24) 20736 activation_29[0][0] __________________________________________________________________________________________________ concatenate_14 (Concatenate) (None, 16, 16, 228) 0 concatenate_13[0][0] conv2d_30[0][0] __________________________________________________________________________________________________ batch_normalization_30 (BatchNo (None, 16, 16, 228) 912 concatenate_14[0][0] __________________________________________________________________________________________________ activation_30 (Activation) (None, 16, 16, 228) 0 batch_normalization_30[0][0] __________________________________________________________________________________________________ conv2d_31 (Conv2D) (None, 16, 16, 96) 21888 activation_30[0][0] __________________________________________________________________________________________________ batch_normalization_31 (BatchNo (None, 16, 16, 96) 384 conv2d_31[0][0] __________________________________________________________________________________________________ activation_31 (Activation) (None, 16, 16, 96) 0 batch_normalization_31[0][0] __________________________________________________________________________________________________ conv2d_32 (Conv2D) (None, 16, 16, 24) 20736 activation_31[0][0] __________________________________________________________________________________________________ concatenate_15 (Concatenate) (None, 16, 16, 252) 0 concatenate_14[0][0] conv2d_32[0][0] __________________________________________________________________________________________________ batch_normalization_32 (BatchNo (None, 16, 16, 252) 1008 concatenate_15[0][0] __________________________________________________________________________________________________ activation_32 (Activation) (None, 16, 16, 252) 0 batch_normalization_32[0][0] __________________________________________________________________________________________________ conv2d_33 (Conv2D) (None, 16, 16, 96) 24192 activation_32[0][0] __________________________________________________________________________________________________ batch_normalization_33 (BatchNo (None, 16, 16, 96) 384 conv2d_33[0][0] __________________________________________________________________________________________________ activation_33 (Activation) (None, 16, 16, 96) 0 batch_normalization_33[0][0] __________________________________________________________________________________________________ conv2d_34 (Conv2D) (None, 16, 16, 24) 20736 activation_33[0][0] __________________________________________________________________________________________________ concatenate_16 (Concatenate) (None, 16, 16, 276) 0 concatenate_15[0][0] conv2d_34[0][0] __________________________________________________________________________________________________ batch_normalization_34 (BatchNo (None, 16, 16, 276) 1104 concatenate_16[0][0] __________________________________________________________________________________________________ activation_34 (Activation) (None, 16, 16, 276) 0 batch_normalization_34[0][0] __________________________________________________________________________________________________ conv2d_35 (Conv2D) (None, 16, 16, 96) 26496 activation_34[0][0] __________________________________________________________________________________________________ batch_normalization_35 (BatchNo (None, 16, 16, 96) 384 conv2d_35[0][0] __________________________________________________________________________________________________ activation_35 (Activation) (None, 16, 16, 96) 0 batch_normalization_35[0][0] __________________________________________________________________________________________________ conv2d_36 (Conv2D) (None, 16, 16, 24) 20736 activation_35[0][0] __________________________________________________________________________________________________ concatenate_17 (Concatenate) (None, 16, 16, 300) 0 concatenate_16[0][0] conv2d_36[0][0] __________________________________________________________________________________________________ batch_normalization_36 (BatchNo (None, 16, 16, 300) 1200 concatenate_17[0][0] __________________________________________________________________________________________________ activation_36 (Activation) (None, 16, 16, 300) 0 batch_normalization_36[0][0] __________________________________________________________________________________________________ conv2d_37 (Conv2D) (None, 16, 16, 96) 28800 activation_36[0][0] __________________________________________________________________________________________________ batch_normalization_37 (BatchNo (None, 16, 16, 96) 384 conv2d_37[0][0] __________________________________________________________________________________________________ activation_37 (Activation) (None, 16, 16, 96) 0 batch_normalization_37[0][0] __________________________________________________________________________________________________ conv2d_38 (Conv2D) (None, 16, 16, 24) 20736 activation_37[0][0] __________________________________________________________________________________________________ concatenate_18 (Concatenate) (None, 16, 16, 324) 0 concatenate_17[0][0] conv2d_38[0][0] __________________________________________________________________________________________________ batch_normalization_38 (BatchNo (None, 16, 16, 324) 1296 concatenate_18[0][0] __________________________________________________________________________________________________ activation_38 (Activation) (None, 16, 16, 324) 0 batch_normalization_38[0][0] __________________________________________________________________________________________________ conv2d_39 (Conv2D) (None, 16, 16, 96) 31104 activation_38[0][0] __________________________________________________________________________________________________ batch_normalization_39 (BatchNo (None, 16, 16, 96) 384 conv2d_39[0][0] __________________________________________________________________________________________________ activation_39 (Activation) (None, 16, 16, 96) 0 batch_normalization_39[0][0] __________________________________________________________________________________________________ conv2d_40 (Conv2D) (None, 16, 16, 24) 20736 activation_39[0][0] __________________________________________________________________________________________________ concatenate_19 (Concatenate) (None, 16, 16, 348) 0 concatenate_18[0][0] conv2d_40[0][0] __________________________________________________________________________________________________ batch_normalization_40 (BatchNo (None, 16, 16, 348) 1392 concatenate_19[0][0] __________________________________________________________________________________________________ activation_40 (Activation) (None, 16, 16, 348) 0 batch_normalization_40[0][0] __________________________________________________________________________________________________ conv2d_41 (Conv2D) (None, 16, 16, 96) 33408 activation_40[0][0] __________________________________________________________________________________________________ batch_normalization_41 (BatchNo (None, 16, 16, 96) 384 conv2d_41[0][0] __________________________________________________________________________________________________ activation_41 (Activation) (None, 16, 16, 96) 0 batch_normalization_41[0][0] __________________________________________________________________________________________________ conv2d_42 (Conv2D) (None, 16, 16, 24) 20736 activation_41[0][0] __________________________________________________________________________________________________ concatenate_20 (Concatenate) (None, 16, 16, 372) 0 concatenate_19[0][0] conv2d_42[0][0] __________________________________________________________________________________________________ batch_normalization_42 (BatchNo (None, 16, 16, 372) 1488 concatenate_20[0][0] __________________________________________________________________________________________________ activation_42 (Activation) (None, 16, 16, 372) 0 batch_normalization_42[0][0] __________________________________________________________________________________________________ conv2d_43 (Conv2D) (None, 16, 16, 186) 69192 activation_42[0][0] __________________________________________________________________________________________________ average_pooling2d_2 (AveragePoo (None, 8, 8, 186) 0 conv2d_43[0][0] __________________________________________________________________________________________________ batch_normalization_43 (BatchNo (None, 8, 8, 186) 744 average_pooling2d_2[0][0] __________________________________________________________________________________________________ activation_43 (Activation) (None, 8, 8, 186) 0 batch_normalization_43[0][0] __________________________________________________________________________________________________ conv2d_44 (Conv2D) (None, 8, 8, 96) 17856 activation_43[0][0] __________________________________________________________________________________________________ batch_normalization_44 (BatchNo (None, 8, 8, 96) 384 conv2d_44[0][0] __________________________________________________________________________________________________ activation_44 (Activation) (None, 8, 8, 96) 0 batch_normalization_44[0][0] __________________________________________________________________________________________________ conv2d_45 (Conv2D) (None, 8, 8, 24) 20736 activation_44[0][0] __________________________________________________________________________________________________ concatenate_21 (Concatenate) (None, 8, 8, 210) 0 average_pooling2d_2[0][0] conv2d_45[0][0] __________________________________________________________________________________________________ batch_normalization_45 (BatchNo (None, 8, 8, 210) 840 concatenate_21[0][0] __________________________________________________________________________________________________ activation_45 (Activation) (None, 8, 8, 210) 0 batch_normalization_45[0][0] __________________________________________________________________________________________________ conv2d_46 (Conv2D) (None, 8, 8, 96) 20160 activation_45[0][0] __________________________________________________________________________________________________ batch_normalization_46 (BatchNo (None, 8, 8, 96) 384 conv2d_46[0][0] __________________________________________________________________________________________________ activation_46 (Activation) (None, 8, 8, 96) 0 batch_normalization_46[0][0] __________________________________________________________________________________________________ conv2d_47 (Conv2D) (None, 8, 8, 24) 20736 activation_46[0][0] __________________________________________________________________________________________________ concatenate_22 (Concatenate) (None, 8, 8, 234) 0 concatenate_21[0][0] conv2d_47[0][0] __________________________________________________________________________________________________ batch_normalization_47 (BatchNo (None, 8, 8, 234) 936 concatenate_22[0][0] __________________________________________________________________________________________________ activation_47 (Activation) (None, 8, 8, 234) 0 batch_normalization_47[0][0] __________________________________________________________________________________________________ conv2d_48 (Conv2D) (None, 8, 8, 96) 22464 activation_47[0][0] __________________________________________________________________________________________________ batch_normalization_48 (BatchNo (None, 8, 8, 96) 384 conv2d_48[0][0] __________________________________________________________________________________________________ activation_48 (Activation) (None, 8, 8, 96) 0 batch_normalization_48[0][0] __________________________________________________________________________________________________ conv2d_49 (Conv2D) (None, 8, 8, 24) 20736 activation_48[0][0] __________________________________________________________________________________________________ concatenate_23 (Concatenate) (None, 8, 8, 258) 0 concatenate_22[0][0] conv2d_49[0][0] __________________________________________________________________________________________________ batch_normalization_49 (BatchNo (None, 8, 8, 258) 1032 concatenate_23[0][0] __________________________________________________________________________________________________ activation_49 (Activation) (None, 8, 8, 258) 0 batch_normalization_49[0][0] __________________________________________________________________________________________________ conv2d_50 (Conv2D) (None, 8, 8, 96) 24768 activation_49[0][0] __________________________________________________________________________________________________ batch_normalization_50 (BatchNo (None, 8, 8, 96) 384 conv2d_50[0][0] __________________________________________________________________________________________________ activation_50 (Activation) (None, 8, 8, 96) 0 batch_normalization_50[0][0] __________________________________________________________________________________________________ conv2d_51 (Conv2D) (None, 8, 8, 24) 20736 activation_50[0][0] __________________________________________________________________________________________________ concatenate_24 (Concatenate) (None, 8, 8, 282) 0 concatenate_23[0][0] conv2d_51[0][0] __________________________________________________________________________________________________ batch_normalization_51 (BatchNo (None, 8, 8, 282) 1128 concatenate_24[0][0] __________________________________________________________________________________________________ activation_51 (Activation) (None, 8, 8, 282) 0 batch_normalization_51[0][0] __________________________________________________________________________________________________ conv2d_52 (Conv2D) (None, 8, 8, 96) 27072 activation_51[0][0] __________________________________________________________________________________________________ batch_normalization_52 (BatchNo (None, 8, 8, 96) 384 conv2d_52[0][0] __________________________________________________________________________________________________ activation_52 (Activation) (None, 8, 8, 96) 0 batch_normalization_52[0][0] __________________________________________________________________________________________________ conv2d_53 (Conv2D) (None, 8, 8, 24) 20736 activation_52[0][0] __________________________________________________________________________________________________ concatenate_25 (Concatenate) (None, 8, 8, 306) 0 concatenate_24[0][0] conv2d_53[0][0] __________________________________________________________________________________________________ batch_normalization_53 (BatchNo (None, 8, 8, 306) 1224 concatenate_25[0][0] __________________________________________________________________________________________________ activation_53 (Activation) (None, 8, 8, 306) 0 batch_normalization_53[0][0] __________________________________________________________________________________________________ conv2d_54 (Conv2D) (None, 8, 8, 96) 29376 activation_53[0][0] __________________________________________________________________________________________________ batch_normalization_54 (BatchNo (None, 8, 8, 96) 384 conv2d_54[0][0] __________________________________________________________________________________________________ activation_54 (Activation) (None, 8, 8, 96) 0 batch_normalization_54[0][0] __________________________________________________________________________________________________ conv2d_55 (Conv2D) (None, 8, 8, 24) 20736 activation_54[0][0] __________________________________________________________________________________________________ concatenate_26 (Concatenate) (None, 8, 8, 330) 0 concatenate_25[0][0] conv2d_55[0][0] __________________________________________________________________________________________________ batch_normalization_55 (BatchNo (None, 8, 8, 330) 1320 concatenate_26[0][0] __________________________________________________________________________________________________ activation_55 (Activation) (None, 8, 8, 330) 0 batch_normalization_55[0][0] __________________________________________________________________________________________________ conv2d_56 (Conv2D) (None, 8, 8, 96) 31680 activation_55[0][0] __________________________________________________________________________________________________ batch_normalization_56 (BatchNo (None, 8, 8, 96) 384 conv2d_56[0][0] __________________________________________________________________________________________________ activation_56 (Activation) (None, 8, 8, 96) 0 batch_normalization_56[0][0] __________________________________________________________________________________________________ conv2d_57 (Conv2D) (None, 8, 8, 24) 20736 activation_56[0][0] __________________________________________________________________________________________________ concatenate_27 (Concatenate) (None, 8, 8, 354) 0 concatenate_26[0][0] conv2d_57[0][0] __________________________________________________________________________________________________ batch_normalization_57 (BatchNo (None, 8, 8, 354) 1416 concatenate_27[0][0] __________________________________________________________________________________________________ activation_57 (Activation) (None, 8, 8, 354) 0 batch_normalization_57[0][0] __________________________________________________________________________________________________ conv2d_58 (Conv2D) (None, 8, 8, 96) 33984 activation_57[0][0] __________________________________________________________________________________________________ batch_normalization_58 (BatchNo (None, 8, 8, 96) 384 conv2d_58[0][0] __________________________________________________________________________________________________ activation_58 (Activation) (None, 8, 8, 96) 0 batch_normalization_58[0][0] __________________________________________________________________________________________________ conv2d_59 (Conv2D) (None, 8, 8, 24) 20736 activation_58[0][0] __________________________________________________________________________________________________ concatenate_28 (Concatenate) (None, 8, 8, 378) 0 concatenate_27[0][0] conv2d_59[0][0] __________________________________________________________________________________________________ batch_normalization_59 (BatchNo (None, 8, 8, 378) 1512 concatenate_28[0][0] __________________________________________________________________________________________________ activation_59 (Activation) (None, 8, 8, 378) 0 batch_normalization_59[0][0] __________________________________________________________________________________________________ conv2d_60 (Conv2D) (None, 8, 8, 96) 36288 activation_59[0][0] __________________________________________________________________________________________________ batch_normalization_60 (BatchNo (None, 8, 8, 96) 384 conv2d_60[0][0] __________________________________________________________________________________________________ activation_60 (Activation) (None, 8, 8, 96) 0 batch_normalization_60[0][0] __________________________________________________________________________________________________ conv2d_61 (Conv2D) (None, 8, 8, 24) 20736 activation_60[0][0] __________________________________________________________________________________________________ concatenate_29 (Concatenate) (None, 8, 8, 402) 0 concatenate_28[0][0] conv2d_61[0][0] __________________________________________________________________________________________________ batch_normalization_61 (BatchNo (None, 8, 8, 402) 1608 concatenate_29[0][0] __________________________________________________________________________________________________ activation_61 (Activation) (None, 8, 8, 402) 0 batch_normalization_61[0][0] __________________________________________________________________________________________________ conv2d_62 (Conv2D) (None, 8, 8, 96) 38592 activation_61[0][0] __________________________________________________________________________________________________ batch_normalization_62 (BatchNo (None, 8, 8, 96) 384 conv2d_62[0][0] __________________________________________________________________________________________________ activation_62 (Activation) (None, 8, 8, 96) 0 batch_normalization_62[0][0] __________________________________________________________________________________________________ conv2d_63 (Conv2D) (None, 8, 8, 24) 20736 activation_62[0][0] __________________________________________________________________________________________________ concatenate_30 (Concatenate) (None, 8, 8, 426) 0 concatenate_29[0][0] conv2d_63[0][0] __________________________________________________________________________________________________ batch_normalization_64 (BatchNo (None, 8, 8, 426) 1704 concatenate_30[0][0] __________________________________________________________________________________________________ activation_64 (Activation) (None, 8, 8, 426) 0 batch_normalization_64[0][0] __________________________________________________________________________________________________ conv2d_65 (Conv2D) (None, 8, 8, 200) 85200 activation_64[0][0] __________________________________________________________________________________________________ conv2d_66 (Conv2D) (None, 8, 8, 200) 360000 conv2d_65[0][0] __________________________________________________________________________________________________ global_average_pooling2d_1 (Glo (None, 200) 0 conv2d_66[0][0] __________________________________________________________________________________________________ softmax_1 (Softmax) (None, 200) 0 global_average_pooling2d_1[0][0] ================================================================================================== Total params: 1,853,736 Trainable params: 1,832,532 Non-trainable params: 21,204 __________________________________________________________________________________________________
In [ ]:
# from keras.datasets import cifar10
# x_train, y_train, x_test, y_test = load_data(resize=False,append=False,shape=(image_dim,image_dim))
In [ ]:
#callbacks
batch_size=64#128
modval=8
max_lr=0.3
augment_data = False
train_generator, validation_generator = create_dataset(image_dim=image_dim,augment=augment_data,batch_size=batch_size)


model_checkpointer=ModelCheckpoint(weights_dir+'large_weights.{epoch:02d}-{val_acc:.2f}.h5', monitor='val_loss',verbose=1, save_best_only=True, save_weights_only=False, mode='min', period=5)
clr = CyclicLR(base_lr=0.1, max_lr=0.2,step_size=8*(int(train_ds_size/batch_size)))
els = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=10, verbose=1, mode='auto', baseline=None, restore_best_weights=False)

use_cosine_annealing=False
lrs = ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=5, min_lr=0.001)
lrs=LearningRateScheduler(cosine_anneal_schedule) if use_cosine_annealing else clr
callbacks = [model_checkpointer,lrs,els,tbc]
initial_epoch=epochs
epochs=10
epochs+=initial_epoch

#h=model.fit(x_train, y_train, batch_size=batch_size, verbose=1,initial_epoch=initial_epoch,epochs=epochs, callbacks=callbacks, validation_data=(x_test,y_test))
h =model.fit_generator(train_generator,steps_per_epoch=int(train_ds_size/batch_size),verbose=1,epochs=epochs,callbacks=callbacks,validation_data=validation_generator,validation_steps=int(validation_ds_size/batch_size))
hist.append(h)
model.save(model_file)
print("Saved model to disk")
/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/image_data_generator.py:346: UserWarning: This ImageDataGenerator specifies `featurewise_std_normalization`, which overrides setting of `featurewise_center`. warnings.warn('This ImageDataGenerator specifies '
Found 100000 images belonging to 200 classes. Found 10000 images belonging to 200 classes. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version. Instructions for updating: Use tf.cast instead. Epoch 1/10
/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/image_data_generator.py:699: UserWarning: This ImageDataGenerator specifies `featurewise_center`, but it hasn't been fit on any training data. Fit it first by calling `.fit(numpy_data)`. warnings.warn('This ImageDataGenerator specifies ' /usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/image_data_generator.py:707: UserWarning: This ImageDataGenerator specifies `featurewise_std_normalization`, but it hasn't been fit on any training data. Fit it first by calling `.fit(numpy_data)`. warnings.warn('This ImageDataGenerator specifies '
57/1562 [>.............................] - ETA: 16:59 - loss: 5.7619 - acc: 0.0074
In [ ]:
#callbacks
batch_size=64#128
modval=8
max_lr=0.3
augment_data = True
train_generator, validation_generator = create_dataset(image_dim=image_dim,augment=augment_data,batch_size=batch_size)


model_checkpointer=ModelCheckpoint(weights_dir+'large_weights.{epoch:02d}-{val_acc:.2f}.h5', monitor='val_loss',verbose=1, save_best_only=True, save_weights_only=False, mode='min', period=5)
clr = CyclicLR(base_lr=0.1, max_lr=0.2,step_size=8*(int(train_ds_size/batch_size)))
els = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=10, verbose=1, mode='auto', baseline=None, restore_best_weights=False)

use_cosine_annealing=False
lrs = ReduceLROnPlateau(monitor='val_loss', factor=0.2,patience=5, min_lr=0.001)
lrs=LearningRateScheduler(cosine_anneal_schedule) if use_cosine_annealing else clr
callbacks = [model_checkpointer,lrs,els,tbc]
initial_epoch=epochs
epochs=20#24
epochs+=initial_epoch

#h=model.fit(x_train, y_train, batch_size=batch_size, verbose=1,initial_epoch=initial_epoch,epochs=epochs, callbacks=callbacks, validation_data=(x_test,y_test))
h =model.fit_generator(train_generator,steps_per_epoch=int(train_ds_size/batch_size),verbose=1,epochs=epochs,callbacks=callbacks,validation_data=validation_generator,validation_steps=int(validation_ds_size/batch_size))
hist.append(h)
model.save(model_file)
print("Saved model to disk")
In [ ]:
print('====================END OF LARGER MODEL====================')