Jovian
Sign In
Learn data science and machine learning by building real-world projects on Jovian

3 months ago
Notebook
Files
Discussions 
Version 2

DeepGlobe Road Extraction - Error Analysis

1. import Packages

! pip install patool --quiet
import os
import patoolib
import pandas as pd
import cv2 as cv
import numpy as np
import tensorflow as tf

from sklearn.model_selection import train_test_split
from skimage.filters import threshold_otsu
from matplotlib import pyplot as plt
from tensorflow.keras.layers import (
    Input,
    Conv2D,
    MaxPooling2D,
    concatenate,
    Conv2DTranspose,
    BatchNormalization,
    Dropout,
    Activation,
    Concatenate
)
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import (
    EarlyStopping,
    ModelCheckpoint
)

2. Load Data

project_path = '/content/drive/MyDrive/AAIC/SCS-2/deep_globe_road_extraction/'
# patoolib.extract_archive(archive=project_path + 'zip_files/DeepGlobe_256.zip', outdir='/content/DeepGlobe')
file_path = '/content/DeepGlobe/metadata.csv'
df = pd.read_csv(filepath_or_buffer=file_path)
train_data = df[df['split'] == 'train']
valid_data = df[df['split'] == 'valid']
test_data = df[df['split'] == 'test']

Selecting a sample data

sample_data = train_data.sample(frac=0.1, random_state=42)
sample_data = sample_data[:100]
sample_data.shape
(100, 4)

3. U-NET Model

Reference → https://youtu.be/GAYJ81M58y8

## Reference → https://youtu.be/GAYJ81M58y8
## The below code is taken from the above video link. Although I have modified it.
## I give full credit to the author of the video and the creators of UNet.

class UNET:
    # convolution block
    def _convolve(self, input_, filters):
        x = Conv2D(filters=filters, kernel_size=(3, 3), kernel_initializer='he_normal', padding='same')(input_)
        x = BatchNormalization()(x)
        x = Activation('relu')(x)
        
        x = Conv2D(filters=filters, kernel_size=(3, 3), kernel_initializer='he_normal', padding='same')(x)
        x = BatchNormalization()(x)
        x = Activation('relu')(x)
        
        return x
    
    # up-sampling and convolotion block
    def _convolve_by_upsampling(self, input_, skip_connector, filters, rate):
        x = Conv2DTranspose(filters=filters, kernel_size=(3, 3), strides=(2, 2), padding='same')(input_)
        x = concatenate([x, skip_connector])
        x = Dropout(rate)(x)
        x = self._convolve(input_=x, filters=filters)
        return x
    
    # UNET main model 
    def unet_main(self, input_, filters=16, rate=0.05):
        # left encoder Path
        c1 = self._convolve(input_=input_, filters=filters)
        p1 = MaxPooling2D(pool_size=(2, 2))(c1)
        p1 = Dropout(rate)(p1)
        
        c2 = self._convolve(input_=p1, filters=filters * 2)
        p2 = MaxPooling2D(pool_size=(2, 2))(c2)
        p2 = Dropout(rate)(p2)
        
        c3 = self._convolve(input_=p2, filters=filters * 4)
        p3 = MaxPooling2D(pool_size=(2, 2))(c3)
        p3 = Dropout(rate)(p3)
        
        c4 = self._convolve(input_=p3, filters=filters * 8)
        p4 = MaxPooling2D(pool_size=(2, 2))(c4)
        p4 = Dropout(rate)(p4)

        # middle bridge
        c5 = self._convolve(input_=p4, filters=filters * 16)
        
        # right decoder path
        c6 = self._convolve_by_upsampling(input_=c5, skip_connector=c4, filters=filters * 8, rate=rate)
        c7 = self._convolve_by_upsampling(input_=c6, skip_connector=c3, filters=filters * 4, rate=rate)
        c8 = self._convolve_by_upsampling(input_=c7, skip_connector=c2, filters=filters * 2, rate=rate)
        c9 = self._convolve_by_upsampling(input_=c8, skip_connector=c1, filters=filters * 1, rate=rate)
        
        output_ = Conv2D(filters=1, kernel_size=(1, 1), activation='sigmoid')(c9)
        
        model = Model(inputs=[input_], outputs=[output_])
        
        return model

4. IoU Score

Credits - pyimagesearch.com

4.1. U-NET IoU Score

def analyze_unet_iou_score(x, model):
    # paths of images
    images_fps   = x['sat_image_path'].to_list()
    # paths of masked images
    masks_fps    = x['mask_path'].to_list()

    images_scores = {}
    for oimg, mimg in zip(images_fps, masks_fps):
        # original image
        simage = plt.imread('/content/DeepGlobe/' + oimg)
        # original mask
        smask = plt.imread('/content/DeepGlobe/' + mimg)
        smask = smask[:,:,0]

        # prediction
        predicted_image  = model.predict(simage[np.newaxis,:,:,:])
        predicted_mask = predicted_image.reshape(simage.shape[0], simage.shape[1])
        # automatic threshold identification
        pmask_thresh = threshold_otsu(predicted_mask)
        # binarizing the mask
        th, predicted_mask = cv.threshold(src=predicted_mask, thresh=pmask_thresh, maxval=255, type=cv.THRESH_BINARY)
        
        bin_thresh = 128
        mask_thresh = predicted_mask > bin_thresh
        union = np.logical_or(smask, mask_thresh)
        intersection = np.logical_and(smask, mask_thresh)
        iou_score = np.sum(intersection) / np.sum(union)
        images_scores[(oimg, mimg)] = iou_score
    
    scores = list(images_scores.values())
    return images_scores, np.mean(scores)

4.2. Basemodel IoU Score

def analyze_basemodel_iou_score(x, model):
    # paths of images
    images_fps   = x['sat_image_path'].to_list()
    # paths of masked images
    masks_fps    = x['mask_path'].to_list()

    images_scores = {}
    for oimg, mimg in zip(images_fps, masks_fps):
        # original image
        simage = cv.imread('/content/DeepGlobe/' + oimg, cv.IMREAD_UNCHANGED)
        simage = cv.cvtColor(simage, cv.COLOR_BGR2RGB)

        # origna mask
        smask = cv.imread('/content/DeepGlobe/' + mimg, cv.IMREAD_UNCHANGED)
        smask = smask[:,:,0]

        # prediction
        predicted_image  = model.predict(simage[np.newaxis,:,:,:])
        predicted_mask = tf.argmax(predicted_image, axis=-1)
        predicted_mask = np.array(predicted_mask[0])
        predicted_mask = np.where((predicted_mask == 1), 255, 0)

        bin_thresh = 128
        mask_thresh = predicted_mask > bin_thresh
        union = np.logical_or(smask, mask_thresh)
        intersection = np.logical_and(smask, mask_thresh)
        iou_score = np.sum(intersection) / np.sum(union)
        images_scores[(oimg, mimg)] = iou_score
    
    scores = list(images_scores.values())
    return images_scores, np.mean(scores)

4.3. Best Medium Worst

4.3.1. U-NET (BMW)
def display_patterns_unet(data_list, model, n=5):
    if not data_list:
        print('No data available.')
        return None
    
    images = [i[0] for i in data_list[:n]]
    masks = [i[1] for i in data_list[:n]]
    
    for oimg, mimg in zip(images, masks):
        # original image
        simage = cv.imread('/content/DeepGlobe/' + oimg, cv.IMREAD_UNCHANGED)
        simage = cv.cvtColor(simage, cv.COLOR_BGR2RGB)

        # origna mask
        smask = cv.imread('/content/DeepGlobe/' + mimg, cv.IMREAD_UNCHANGED)
        smask = smask[:,:,0]

        # prediction
        predicted_image  = model.predict(simage[np.newaxis,:,:,:])
        predicted_mask = predicted_image.reshape(simage.shape[0], simage.shape[1])
        # automatic threshold identification
        pmask_thresh = threshold_otsu(predicted_mask)
        # binarizing the mask
        th, predicted_mask = cv.threshold(src=predicted_mask, thresh=pmask_thresh, maxval=255, type=cv.THRESH_BINARY)

        # titles
        image_title = oimg.split('/')[-1]
        mask_title = mimg.split('/')[-1]
        
        # plotting figure
        plt.figure(figsize=(15, 6))
        
        plt.subplot(131)
        plt.axis("off")
        plt.title(image_title)
        plt.imshow(simage)

        plt.subplot(132)
        plt.axis("off")
        plt.title(mask_title)
        plt.imshow(smask, cmap='gray')

        plt.subplot(133)
        plt.axis("off")
        plt.title("Prediction - {}".format(image_title))
        plt.imshow(predicted_mask, cmap='gray')
        
        plt.show()

    return None
4.3.2. Basemodels (BMW)
def display_patterns_basemodel(data_list, model, n=5):
    if not data_list:
        print('No data available.')
        return None
    
    images = [i[0] for i in data_list[:n]]
    masks = [i[1] for i in data_list[:n]]
    
    for oimg, mimg in zip(images, masks):
        # original image
        simage = cv.imread('/content/DeepGlobe/' + oimg, cv.IMREAD_UNCHANGED)
        simage = cv.cvtColor(simage, cv.COLOR_BGR2RGB)

        # origna mask
        smask = cv.imread('/content/DeepGlobe/' + mimg, cv.IMREAD_UNCHANGED)
        smask = smask[:,:,0]
        
        # prediction
        predicted_image  = model.predict(simage[np.newaxis,:,:,:])
        predicted_mask = tf.argmax(predicted_image, axis=-1)
        predicted_mask = np.array(predicted_mask[0])
        predicted_mask = np.where((predicted_mask == 1), 255, 0)

        # titles
        image_title = oimg.split('/')[-1]
        mask_title = mimg.split('/')[-1]
        
        # plotting figure
        plt.figure(figsize=(15, 6))
        
        plt.subplot(131)
        plt.axis("off")
        plt.title(image_title)
        plt.imshow(simage)

        plt.subplot(132)
        plt.axis("off")
        plt.title(mask_title)
        plt.imshow(smask, cmap='gray')

        plt.subplot(133)
        plt.axis("off")
        plt.title("Prediction - {}".format(image_title))
        plt.imshow(predicted_mask, cmap='gray')
        
        plt.show()

    return None
4.3.3. Segregation
def segregate_by_iou(sm_data_dictionary):
    # best thresh → (IoU >= 0.5)
    # medium thresh → (IoU >= 0.25) or (IoU < 0.5)
    # worst thresh → (IoU < 0.25)

    best = []
    medium = []
    worst = []

    for (t, i) in sm_data_dictionary.items():
        if (i >= 0.5):
            best.append(t)
        elif (i >= 0.25) or (i < 0.5):
            medium.append(t)
        else:
            worst.append(t)
    
    return best, medium, worst

5. Load U-NET Model

def get_unet_model(model_name):
    u = UNET()
    inputs = Input(shape=(256, 256, 3))
    model = u.unet_main(input_=inputs)
    model.compile(optimizer='Adam', loss='binary_crossentropy', metrics=['accuracy'])

    model_path = project_path + 'models/{}.h5'.format(model_name)
    if not os.path.isfile(path=model_path):
        print('Please train the model first.')
    else:
        model.load_weights(model_path)
    
    return model

5.1. U-NET IoU Score

5.1.1. No Augmentation
unet_scratch_name = 'unet_scratch'
unet_noaug_model = get_unet_model(model_name=unet_scratch_name)
unet_noaug_sm, unet_noaug_iou = analyze_unet_iou_score(x=sample_data, model=unet_noaug_model)
print('The IoU for the sample data is : {}'.format(unet_noaug_iou))
The IoU for the sample data is : 0.5126380687564733

Data segregation

us_best, us_medium, us_worst = segregate_by_iou(unet_noaug_sm)

Best images - patterns

  • We can observe that the prediction and original mask are of exact match.
  • In fact, in some cases the prediction masks contain the extact road path than of original masks given.
  • These are the images that have high IoU scores.
display_patterns_unet(data_list=us_best, model=unet_noaug_model)
Notebook Image
Notebook Image
Notebook Image