# Setting the path to my local directory where my dataset is placed

import os
os.chdir('E:\\Finale Project\\Python Project')
print (os.getcwd())

# Importing necessary libraries
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import seaborn as sns
from sklearn import preprocessing,metrics 
%matplotlib inline
from IPython.display import Image
from matplotlib import rcParams
rcParams['figure.figsize'] = 11,6

# Loading the data from txt file
data = pd.read_csv('XYZCorp_LendingData.txt',sep="\t",low_memory=False)
data.info()
data.describe()

# Plot Loan Status
plt.figure(figsize= (12,6))
plt.ylabel('Loan Status')
plt.xlabel('Count')
data['default_ind'].value_counts().plot(kind = 'barh', grid = True)
plt.show()

# Analyzing Purpose
total = len(data)

plt.figure(figsize = (14,6))

g = sns.countplot(x="", data=data, 
                  color='blue')
g.set_xticklabels(g.get_xticklabels(),rotation=45)
g.set_xlabel("Loan Status Categories", fontsize=12)
g.set_ylabel("Count", fontsize=15)
g.set_title("Loan Status Types Distribution", fontsize=20)
sizes=[]
for p in g.patches:
    height = p.get_height()
    sizes.append(height)
    g.text(p.get_x()+p.get_width()/2.,
            height + 3,
            '{:1.2f}%'.format(height/total*100),
            ha="center", fontsize=12) 
g.set_ylim(0, max(sizes) * 1.10)

plt.show()

data.default_ind.value_counts()

# Data Dimension
data.shape

# Drop these features for now
data.drop([    'id',
             'member_id',
             'emp_title',
             'title',
#              'url',
             'zip_code',
             'verification_status',
             'home_ownership',
             'issue_d',
             'earliest_cr_line',
             'last_pymnt_d',
             'next_pymnt_d',
             'desc',
#             'pymnt_plan',
#             'initial_list_status',
#             'addr_state',
             'last_credit_pull_d', 
                                    ], axis=1, inplace=True)

data.drop(['verification_status_joint'], axis=1, inplace=True)

# Show records number
data.count().sort_values()

# Drop columns with less than 25% data.
lack_of_data_idx = [x for x in data.count() < 855969*0.25]
data.drop(data.columns[lack_of_data_idx], 1, inplace=True)

# After Deletion
data.info()

print (data.mths_since_last_delinq.min(), data.mths_since_last_delinq.max())
print(data.mths_since_last_delinq.mean())
print(data.mths_since_last_delinq.median())

data.mths_since_last_delinq = data.mths_since_last_delinq.fillna(data.mths_since_last_delinq.median())

data.dropna(inplace=True)

data.info()

# Calculate Good and Bad Loan Status Ratio
good_loan =  len(data[(data.default_ind == 0)])
print ('Good/Bad Loan Ratio: %.2f%%'  % (good_loan/len(data)*100))

# create an bad/good loan indicator feature
data['good_loan'] = np.where((data.default_ind == 0) , 1, 0)

# Hot encode some categorical features 
columns = ['term', 'grade', 'sub_grade', 'emp_length', 'purpose', 'application_type','addr_state',
           'pymnt_plan', 'initial_list_status']

for col in columns:
    tmp_df = pd.get_dummies(data[col], prefix=col)
    data = pd.concat((data, tmp_df), axis=1)

# drop attributes that we hot-encoded
data.drop([#'loan_status',
           'term',
           'grade',
           'sub_grade',
           'emp_length',
           'addr_state',
           'initial_list_status',
           'pymnt_plan',
           'purpose',
           'application_type'], axis=1, inplace=True)

# drop attributes that we hot-encoded
data.drop(['default_ind'], axis=1, inplace=True)

# Rename some features to concur w/ some algorithms
data = data.rename(columns= {'emp_length_< 1 year':'emp_length_lt_1 year',
                         'emp_length_n/a':'emp_length_na'})

# Due to resource limitation, we limit data to only the first 10,000 records.
data = data[:10000]

data.head()

# Split Train/Test data
from sklearn.model_selection import train_test_split

y = data['good_loan']
X = data.ix[:, data.columns != 'good_loan']

X_train, X_test, y_train, y_test = train_test_split (X, y, test_size=0.2, random_state=44)

!pip install sklearn

# Bring in evaluator
import sklearn.metrics as mt
from sklearn.model_selection import cross_val_score

# Flatten Data
from sklearn.preprocessing import StandardScaler, RobustScaler

#std_scaler = StandardScaler()
rob_scaler = RobustScaler()

#X_train_S = std_scaler.fit_transform(X_train)
#X_test_S = std_scaler.transform(X_test)

# Use robust scaler to reduce outliers
X_train_R = rob_scaler.fit_transform(X_train)
X_test_R = rob_scaler.transform(X_test)

from sklearn.svm import SVC

# Weighted prediction feature
y_0 = len(y_train[y_train == 0])/len(y_train)
y_1 = 1 - y_0

svm_clf = SVC(class_weight={0:y_1, 1:y_0})
svm_clf.fit(X_train_R, y_train)

svm_predictions = svm_clf.predict(X_test_R) # Save prediction


#print(svm_clf.score(X_test_R, y_test))
scores = cross_val_score(svm_clf, X_test_R, y_test, cv=5)
print(scores)
print("Accuracy: %0.2f (+/- %0.2f)\n" % (scores.mean(), scores.std() * 2))

print(mt.classification_report(y_test, svm_predictions))
print(mt.confusion_matrix(y_test, svm_predictions))

!pip install joblib

# SVM - SMOTE

from imblearn.over_sampling import SMOTE

sm = SMOTE(k_neighbors=10, random_state=44, kind = 'svm')
X_res_train, y_res_train = sm.fit_sample(X_train_R, y_train)

svm_sm_clf = SVC()
svm_sm_clf.fit(X_res_train, y_res_train)

svm_sm_predictions = svm_clf.predict(X_test_R)

#print(svm_sm_clf.score(X_test_R, y_test))
scores = cross_val_score(svm_sm_clf, X_test_R, y_test, cv=5)
print(scores)
print("Accuracy: %0.2f (+/- %0.2f)\n" % (scores.mean(), scores.std() * 2))

print(mt.classification_report(y_test, svm_sm_predictions))
print(mt.confusion_matrix(y_test, svm_sm_predictions))

from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_estimators = 20)
rf.fit(X_train, y_train)
       
rf_predictions = rf.predict(X_test)

#print(rf.score(X_test, y_test))
scores = cross_val_score(rf, X_test, y_test, cv=5)
print(scores)
print("Accuracy: %0.2f (+/- %0.2f)\n" % (scores.mean(), scores.std() * 2))

print(mt.classification_report(y_test, rf_predictions))
print(mt.confusion_matrix(y_test, rf_predictions))

!pip install xgboost

from xgboost import XGBClassifier

xgb = XGBClassifier()
xgb.fit(X_train, y_train)

xgb_predictions = xgb.predict(X_test)
                            
#print(xgb.score(X_test, y_test))
scores = cross_val_score(xgb, X_test, y_test, cv=5)
print(scores)
print("Accuracy: %0.2f (+/- %0.2f)\n" % (scores.mean(), scores.std() * 2))

print(mt.classification_report(y_test, xgb_predictions))
print(mt.confusion_matrix(y_test, xgb_predictions))

# Taking the backup of loaded dataset
dataset_backup=data.copy()
dataset_backup.shape

missingdata = [x for x in data.count() < len(data)*0.35]
data.drop(data.columns[missingdata], axis=1, inplace=True)
data.shape
data.columns
len(data.columns)

# Checking and dropping columns with just one unique value
unique = data.nunique()
unique = unique[unique.values == 1]
data.drop(labels = list(unique.index), axis =1, inplace=True)

# Checking for duplicates
data.duplicated().value_counts()

len(data.columns)

data['annual_inc'].describe()

# Checking for outliers Data types for numeric columns
data['annual_inc']= data['annual_inc'].astype(float)
data['annual_inc'].describe()
data_fin=data.drop(data[data.annual_inc>1e+05].index)

# Bucketing and dropping extra columns

data_fin['int_rate']=data_fin['int_rate'].astype(str)
data_fin['int_rate']= data_fin['int_rate'].map(lambda x: x.rstrip('%'))
data_fin['int_rate']= data_fin['int_rate'].astype(float)
data_fin['int_rate'].describe() 
buck = [0, 5, 10, 15, 20,25, 35]
lab = ['0-5', '5-10', '10-15', '15-20', '20-25','>25']
data_fin['int_rate_range'] = pd.cut(data_fin['int_rate'], buck, labels=lab)

data_fin['loan_amnt'].describe() #0-40k
buck = [0, 5000, 10000, 15000, 20000, 25000,40000]
lab = ['0-5000', '5000-10000', '10000-15000', '15000-20000', '20000-25000','25000 and above']
data_fin['loan_amnt_range'] = pd.cut(data_fin['loan_amnt'], buck, labels=lab)

data_fin['annual_inc'].describe() #range 1 to 1 mill
buck = [0, 25000, 50000, 75000, 100000,1000000]
lab = ['0-25000', '25000-50000', '50000-75000', '75000-100000', '100000 and above']
data_fin['annual_inc_range'] = pd.cut(data_fin['annual_inc'], buck, labels=lab)

sns.distplot(data_fin['loan_amnt'])

sns.distplot(data_fin['int_rate'])

sns.distplot(data_fin['annual_inc'])

sns.countplot(data_fin['default_ind'])
data_fin['default_ind'].value_counts()

Well, Looking at this bar plot we can say this is case of class imbalance problem. 

data_fin['default_ind'].value_counts()/len(data_fin)

sns.countplot(data_fin['default_ind'])
sns.countplot(data_fin['purpose'],hue=data_fin['default_ind'])
sns.countplot(data_fin['purpose'],hue=data_fin['loan_amnt_range'])

Home ownership variable - ANY and NONE levels do not signify anything so can be dropped

data_fin.drop(data_fin[data_fin['home_ownership']== 'ANY'].index, inplace=True)
data_fin.drop(data_fin[data_fin['home_ownership']== 'NONE'].index, inplace=True)
sns.countplot(data_fin['home_ownership'],hue=data_fin['default_ind'])

#employment length
sns.countplot(data_fin['emp_length'],hue=data_fin['default_ind'])
sns.countplot(data_fin['emp_length'],hue=data_fin['loan_amnt_range'])

#geography
sns.countplot(data_fin['addr_state'],hue=data_fin['default_ind']) 
sns.countplot(data_fin['addr_state'],hue=data_fin['loan_amnt_range'])

# monthly trend
data_fin['issue_yr']=pd.DatetimeIndex(data_fin['issue_d']).year
data_fin['issue_mon']=pd.DatetimeIndex(data_fin['issue_d']).month
sns.countplot(data_fin['issue_mon'],hue=data_fin['default_ind'])

Checking correlation between a few key business variables

cor_loan=data_fin[['loan_amnt','annual_inc', 'default_ind', 'int_rate', 'dti', 
               'tot_cur_bal', 'funded_amnt']]
f, ax = plt.subplots(figsize=(16, 9))
sns.heatmap(cor_loan.corr(), 
            xticklabels=cor_loan.columns.values,
            yticklabels=cor_loan.columns.values,annot= True)
plt.show()

Variable selection: Dropping retrospective variables that would not have been known at the time of loan issuance 
and thus can skew the prediction and some other unnecessary features. 

data_fin.drop(['id', 'member_id','zip_code' ,'emp_title' ,'funded_amnt', 'funded_amnt_inv', 'total_pymnt', 
               'total_pymnt_inv', 'total_rec_prncp', 'total_rec_int', 'out_prncp','out_prncp_inv', 
               'last_pymnt_d', 'last_pymnt_amnt', 'last_credit_pull_d', 'collections_12_mths_ex_med',
               'recoveries', 'collection_recovery_fee','title','revol_util'], axis=1, inplace=True)

# Dropping some more unnecessary features
data_fin.drop(['next_pymnt_d', 'tot_coll_amt' , 'tot_cur_bal' ,'total_rev_hi_lim'], axis=1, inplace=True)

len(data_fin.columns)

For categorical variables- Hot encoding used wherever clear levels could not be established
'emp_length'

data_fin['emp_length'].value_counts()
emp_range= {'< 1 year':0.5, '1 year':1, '2 years': 2, '3 years':3,
            '4 years':4, '5 years':5,'6 years':6,'7 years':7,
            '8 years':8,'9 years':9, '10+ years':10}
data_fin['emplen'] = data_fin["emp_length"].map(emp_range)
data_fin['emplen'].isnull().sum() 
data_fin['emplen'].value_counts()

Categorical: 'term', 'grade', 'emp_length', 'home_ownership', 'verification_status', 'issue_d', 'purpose', 'initial_list_status', 'application_type'

Continuous: 'loan_amnt', 'funded_amnt', 'funded_amnt_inv', 'int_rate', 'installment', 'annual_inc', 'dti', 'delinq_2yrs','open_acc', 'pub_rec', 'revol_bal', 'revol_util', 'total_acc', 'out_prncp', 'out_prncp_inv', 'total_pymnt', 'total_pymnt_inv', 'total_rec_prncp', 'total_rec_int', 'total_rec_late_fee', 'recoveries', 'collection_recovery_fee', 'last_pymnt_amnt', 'collections_12_mths_ex_med', 'tot_coll_amt', 'tot_cur_bal', 'total_rev_hi_lim'

Count:

Missing value imputation

#Checking missing values in all the columns - how many null in each column
data_fin.isna()
data_fin.isnull().sum()

nullseries=pd.isnull(data_fin).sum()
nullseries[nullseries>0]

data_fin['emplen'] = data_fin['emplen'].replace(np.nan, 10)
data_fin.drop(['emp_length'],axis=1,inplace=True)

data_fin['mths_since_last_delinq'] = data_fin['mths_since_last_delinq'].fillna(data_fin['mths_since_last_delinq'].median()) #mean and median v similar

#very few missing values for all of them
#data_fin['dti'] = data_fin['dti'].fillna(data_fin['dti'].mean())
#data_fin['inq_last_6mths'] = data_fin['inq_last_6mths'].fillna(data_fin['inq_last_6mths'].mean())
#data_fin['collections_12_mths_ex_med'] = data_fin['collections_12_mths_ex_med'].fillna(data_fin['collections_12_mths_ex_med'].mean())

# A lot of NAs- with integer values so median replacement
# NONE

data_fin.isna()
data_fin.isnull().sum()

Feature Engineering- Encoding where not ordinality could not be established. Variables such as- verification status,subgrade, purpose,addr_state. Hot encoding needs to be done for purpose.

verification_map={'Source Verified':3, 'Verified':2, 'Not Verified':1}
data_fin['verification_status']=data_fin['verification_status'].map(verification_map)

ownership_map={'MORTGAGE':1, 'RENT':2, 'OWN':3, 'OTHER':4}
data_fin['home_ownership']=data_fin['home_ownership'].map(ownership_map)

subgrade_map={'A1':1,'A2':2, 'A3':3, 'A4':4, 'A5':5, 'B1':6, 'B2':7, 'B3':8, 'B4':9, 'B5':10, 
              'C1':11, 'C2':12, 'C3':13, 'C4':14, 'C5':15, 'D1':16, 'D2':17, 'D3':18, 
              'D4':19, 'D5':20, 'E1':21, 'E2':22, 'E3':23, 'E4':24, 'E5':25, 'F1':26, 
              'F2':27, 'F3':28, 'F4':29, 'F5':30, 'G1':31, 'G2':32, 'G3':33, 'G4':34, 'G5':35}
data_fin['sub_grade']=data_fin['sub_grade'].map(subgrade_map)

data_fin= data_fin[data_fin['purpose'] != 'educational']
data_fin= data_fin[data_fin['purpose'] !='wedding']
data_fin= data_fin[data_fin['purpose'] !='other']

enc1= pd.get_dummies(data_fin['purpose'])
data_fin=pd.concat((data_fin,enc1), axis=1)
data_fin.drop(['purpose'],axis=1,inplace=True)

enc2= pd.get_dummies(data_fin['addr_state'])
data_fin=pd.concat((data_fin,enc2), axis=1)
data_fin.drop(['addr_state'],axis=1,inplace=True)

enc4= pd.get_dummies(data_fin['application_type'])
data_fin=pd.concat((data_fin,enc4), axis=1)
data_fin.drop(['application_type'],axis=1,inplace=True)

enc5= pd.get_dummies(data_fin['term'])
data_fin=pd.concat((data_fin,enc5), axis=1)
data_fin.drop(['term'],axis=1,inplace=True)

data_fin.columns

data_fin.drop(['earliest_cr_line','loan_amnt_range', 'annual_inc_range','int_rate_range','grade',
               'issue_yr','issue_mon','pymnt_plan','initial_list_status', 'total_rec_late_fee'],axis=1, inplace=True)
data_fin.isna()
data_fin.isnull().sum()

len(data_fin.columns)

# emp_length
emp_length_count=data_fin['emplen'].value_counts()
print(emp_length_count)
bp=sns.barplot(emp_length_count.index,emp_length_count.values)
bp.set_xticklabels(bp.get_xticklabels(), rotation=45)

# home_ownership
home_ownership_count=data_fin['home_ownership'].value_counts()
print(home_ownership_count)
sns.barplot(home_ownership_count.index,home_ownership_count.values)

# acc_now_delinq
acc_now_delinq_count=data_fin['acc_now_delinq'].value_counts()
print(acc_now_delinq_count)
sns.barplot(acc_now_delinq_count.index,acc_now_delinq_count.values)

# Since only one class dominates so we drop this feature
data_fin.drop(['acc_now_delinq'],axis=1,inplace=True)

# annual income
plt.figure(1)
plt.subplot(121)
sns.distplot(data_fin['annual_inc']);

plt.subplot(122)
data_fin['annual_inc'].plot.box(figsize=(16,5))

plt.show()

# dti
plt.figure(1)
plt.subplot(121)
sns.distplot(data_fin['dti']);

plt.subplot(122)
data_fin['dti'].plot.box(figsize=(16,5))

plt.show()

# Outliers Treatment
#Find mean of the column "dti"
dti_mean = int(data_fin['dti'].mean())

#FInd 75th Percentile of the column "dti"
IQR_dti_P75 = data_fin['dti'].quantile(q=0.75)

#FInd 25th Percentile of the column "dti"
IQR_dti_P25 = data_fin['dti'].quantile(q=0.25)

#FInd IQR of the column "dti"
IQR_dti = IQR_dti_P75-IQR_dti_P25

#Fix boundaries to detect outliers in column "dti"
IQR_LL = int(IQR_dti_P25 - 1.5*IQR_dti)
IQR_UL = int(IQR_dti_P75 + 1.5*IQR_dti)

#treating upper end outier with mean
data_fin.loc[data_fin['dti']>IQR_UL , 'dti'] = dti_mean

#treating lower end outlier as mean
data_fin.loc[data_fin['dti']<IQR_LL , 'dti'] = dti_mean

plt.figure(1)
plt.subplot(121)
sns.distplot(data_fin['delinq_2yrs']);

plt.subplot(122)
data_fin['delinq_2yrs'].plot.box(figsize=(16,5))

plt.show()

# Outliers Treatment
#Find mean of the column "delinq_2yrs"
delinq_2yrs_mean = int(data_fin['delinq_2yrs'].mean())

#FInd 75th Percentile of the column "delinq_2yrs"
IQR_delinq_2yrs_P75 = data_fin['delinq_2yrs'].quantile(q=0.75)

#FInd 25th Percentile of the column "delinq_2yrs"
IQR_delinq_2yrs_P25 = data_fin['delinq_2yrs'].quantile(q=0.25)

#FInd IQR of the column "dti"
IQR_delinq_2yrs = IQR_delinq_2yrs_P75-IQR_delinq_2yrs_P25

#Fix boundaries to detect outliers in column "dti"
IQR_LL = int(IQR_delinq_2yrs_P25 - 1.5*IQR_delinq_2yrs)
IQR_UL = int(IQR_delinq_2yrs_P75 + 1.5*IQR_delinq_2yrs)

#treating upper end outier with mean
data_fin.loc[data_fin['delinq_2yrs']>IQR_UL , 'delinq_2yrs'] = dti_mean

#treating lower end outlier as mean
data_fin.loc[data_fin['delinq_2yrs']<IQR_LL , 'delinq_2yrs'] = dti_mean

# open_acc
plt.figure(1)
plt.subplot(121)
sns.distplot(data_fin['open_acc']);

plt.subplot(122)
data_fin['open_acc'].plot.box(figsize=(16,5))

plt.show()

# Outliers Treatment
#Find mean of the column "open_acc"
open_acc_mean = int(data_fin['open_acc'].mean())

#FInd 75th Percentile of the column "open_acc"
IQR_open_acc_P75 = data_fin['open_acc'].quantile(q=0.75)

#FInd 25th Percentile of the column "open_acc"
IQR_open_acc_P25 = data_fin['open_acc'].quantile(q=0.25)

#FInd IQR of the column "open_acc"
IQR_open_acc = IQR_open_acc_P75-IQR_open_acc_P25

#Fix boundaries to detect outliers in column "open_acc"
IQR_LL = int(IQR_open_acc_P25 - 1.5*IQR_open_acc)
IQR_UL = int(IQR_open_acc_P75 + 1.5*IQR_open_acc)

#treating upper end outier with mean
data_fin.loc[data_fin['open_acc']>IQR_UL , 'open_acc'] = open_acc_mean

#treating lower end outlier as mean
data_fin.loc[data_fin['open_acc']<IQR_LL , 'open_acc'] = open_acc_mean

# pub_rec
plt.figure(1)
plt.subplot(121)
sns.distplot(data_fin['pub_rec']);

plt.subplot(122)
data_fin['pub_rec'].plot.box(figsize=(16,5))

plt.show()

# Outliers Treatment
#Find mean of the column "pub_rec"
pub_rec_mean = int(data_fin['pub_rec'].mean())

#FInd 75th Percentile of the column "pub_rec"
IQR_pub_rec_P75 = data_fin['pub_rec'].quantile(q=0.75)

#FInd 25th Percentile of the column "pub_rec"
IQR_pub_rec_P25 = data_fin['pub_rec'].quantile(q=0.25)

#FInd IQR of the column "pub_rec"
IQR_pub_rec = IQR_pub_rec_P75-IQR_pub_rec_P25

#Fix boundaries to detect outliers in column "pub_rec"
IQR_LL = int(IQR_pub_rec_P25 - 1.5*IQR_pub_rec)
IQR_UL = int(IQR_pub_rec_P75 + 1.5*IQR_pub_rec)

#treating upper end outier with mean
data_fin.loc[data_fin['pub_rec']>IQR_UL , 'pub_rec'] = pub_rec_mean

#treating lower end outlier as mean
data_fin.loc[data_fin['pub_rec']<IQR_LL , 'pub_rec'] = pub_rec_mean

plt.figure(1)
plt.subplot(121)
sns.distplot(data_fin['revol_bal']);

plt.subplot(122)
data_fin['revol_bal'].plot.box(figsize=(16,5))

plt.show()

# Outliers Treatment
#Find mean of the column "revol_bal"
revol_bal_mean = int(data_fin['revol_bal'].mean())

#FInd 75th Percentile of the column "revol_bal"
IQR_revol_bal_P75 = data_fin['revol_bal'].quantile(q=0.75)

#FInd 25th Percentile of the column "revol_bal"
IQR_revol_bal_P25 = data_fin['revol_bal'].quantile(q=0.25)

#FInd IQR of the column "revol_bal"
IQR_revol_bal = IQR_revol_bal_P75-IQR_revol_bal_P25

#Fix boundaries to detect outliers in column "revol_bal"
IQR_LL = int(IQR_revol_bal_P25 - 1.5*IQR_revol_bal)
IQR_UL = int(IQR_revol_bal_P75 + 1.5*IQR_revol_bal)

#treating upper end outier with mean
data_fin.loc[data_fin['revol_bal']>IQR_UL , 'revol_bal'] = revol_bal_mean

#treating lower end outlier as mean
data_fin.loc[data_fin['revol_bal']<IQR_LL , 'revol_bal'] = revol_bal_mean

# total_acc
plt.figure(1)
plt.subplot(121)
sns.distplot(data_fin['total_acc']);

plt.subplot(122)
data_fin['total_acc'].plot.box(figsize=(16,5))

plt.show()

# Outliers Treatment
#Find mean of the column "total_acc"
total_acc_mean = int(data_fin['total_acc'].mean())

#FInd 75th Percentile of the column "total_acc"
IQR_total_acc_P75 = data_fin['total_acc'].quantile(q=0.75)

#FInd 25th Percentile of the column "total_acc"
IQR_total_acc_P25 = data_fin['total_acc'].quantile(q=0.25)

#FInd IQR of the column "total_acc"
IQR_total_acc = IQR_total_acc_P75-IQR_total_acc_P25

#Fix boundaries to detect outliers in column "total_acc"
IQR_LL = int(IQR_total_acc_P25 - 1.5*IQR_total_acc)
IQR_UL = int(IQR_total_acc_P75 + 1.5*IQR_total_acc)

#treating upper end outier with mean
data_fin.loc[data_fin['total_acc']>IQR_UL , 'total_acc'] = total_acc_mean

#treating lower end outlier as mean
data_fin.loc[data_fin['total_acc']<IQR_LL , 'total_acc'] = total_acc_mean

data_fin.columns.all

# Categorizing Numeric and count data

data_numeric=data_fin.loc[:,['loan_amnt','int_rate','sub_grade', 'home_ownership','verification_status','inq_last_6mths',
       'annual_inc', 'dti', 'delinq_2yrs','open_acc','mths_since_last_delinq',  
       'total_acc', 'emplen', 'revol_bal',
       ]]

#calculating correlation among numeric variable 
corr_matrix = data_numeric.corr() 

#plot correlation matrix
plt.figure(figsize=(20,12))
sns.heatmap(corr_matrix,
            cmap='coolwarm',
            annot=True);

Sampling procedures- 
1. SMOTE: can increase recall at the cost of precision 
2. Undersampling: if less data overall, minority class gets you less data 
3. ADASYN will focus on samples which are difficult to classify with NN

data_fin['issue_d'] = pd.to_datetime(data_fin['issue_d'])
data_fin = data_fin.set_index(data_fin['issue_d'])
data_fin = data_fin.sort_index()

train = data_fin['June 2007':'May 2015']
test  = data_fin['June 2015':'Dec 2015']

print('Train Dataset:',train.shape)
print('Test Dataset:',test.shape)

train =train.drop('issue_d' , axis=1)
test =test.drop('issue_d', axis=1)

from collections import Counter
#from imblearn.under_sampling import RandomUnderSampler
from sklearn.model_selection import StratifiedKFold

from imblearn.over_sampling import ADASYN, RandomOverSampler, SMOTE 
from imblearn.under_sampling import RandomUnderSampler

X_train = train.drop('default_ind', axis=1)
y_train = train.loc[:,['default_ind']]
X_test  = test.drop('default_ind', axis=1)
y_test  = test.loc[:,['default_ind']]

 
#OVERSAMPLING-SMOTE
sm= SMOTE(random_state=42)
X_sm, y_sm = sm.fit_sample(X_train, y_train)

# #OVERSAMPLING-RANDOM
# ros= RandomOverSampler(random_state=555)
# X_over, y_over= ros.fit_sample(X_train, y_train)

# #Undersampling
# rus = RandomUnderSampler(return_indices=True, random_state=555)
# X_resampled, y_resampled, idx_resampled= rus.fit_sample(X_train, y_train)

data_fin.info()
data_fin.describe()

from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_std= scaler.fit_transform(X_sm)
X_std_test= scaler.fit_transform(X_test)

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, roc_auc_score, f1_score,classification_report

#1. Logistic Regression

#SMOTE
lr_sm = LogisticRegression() 
lr_sm.fit(X_sm, y_sm)
lr_sm.score(X_sm, y_sm)
y_pred_sm= lr_sm.predict(X_test)
accuracy_score(y_test, y_pred_sm)
roc_auc_score(y_test, y_pred_sm)
classification_report(y_test, y_pred_sm)
f1_score(y_test, y_pred_sm)

lr_sm.coef_.shape

#2 Decision Tree
from sklearn.tree import tree, DecisionTreeClassifier
dt = tree.DecisionTreeClassifier(criterion='gini')
dt.fit(X_sm, y_sm)
dt.score(X_sm, y_sm)
y_pred_sm= dt.predict(X_test)
accuracy_score(y_test, y_pred_sm)
roc_auc_score(y_test, y_pred_sm)
classification_report(y_test, y_pred_sm)
f1_score(y_test, y_pred_sm)    

#3 Random Forests
from sklearn.ensemble import RandomForestClassifier 
rf = RandomForestClassifier(n_estimators=80, max_features= 'log2')
rf.fit(X_sm, y_sm)
#rf.score(X_sm, y_sm)
y_rf= rf.predict(X_test)
accuracy_score(y_test, y_rf)
roc_auc_score(y_test, y_rf)
classification_report(y_test, y_rf)
f1_score(y_test, y_rf)    

#tuning RF
from sklearn.grid_search import GridSearchCV
param_grid = { 
    'n_estimators': [20, 80],
    'max_features': [None, 'log2', 'sqrt']
}
CV_rf = GridSearchCV(estimator=rf, param_grid=param_grid, cv= 5)
CV_rf.fit(X_sm, y_sm)
print (CV_rf.best_params)

#variable importance
importances = rf.feature_importances_
std = np.std([tree.feature_importances_ for tree in rf.estimators_],
             axis=0)
indices = np.argsort(importances)[::-1]

print("Feature ranking:")

for f in range(X.shape[1]):
    print("%d. feature %d (%f)" % (f + 1, indices[f], importances[indices[f]]))

plt.figure()
plt.title("Feature importances")
plt.bar(range(X.shape[1]), importances[indices],
       color="r", yerr=std[indices], align="center")
plt.xticks(range(X.shape[1]), indices)
plt.xlim([-1, X.shape[1]])
plt.show()

# Gradient Boosted Trees
from sklearn.ensemble import GradientBoostingClassifier  
gb = GradientBoostingClassifier(n_estimators=80, learning_rate=1, 
                                random_state=42)
gb.fit(X_sm, y_sm)
#rf.score(X_sm, y_sm)
y_gb= gb.predict(X_test)
#accuracy_score(y_test, y_rf)
roc_auc_score(y_test, y_gb)
classification_report(y_test, y_gb)
f1_score(y_test, y_gb) 

# Support Vector Machines
from sklearn import svm
from sklearn.svm import SVC

model_svm = svm.SVC(random_state=42, tol=100,class_weight='balanced')
model_svm.fit(X_std, y_sm)
y_svm= model_svm.predict(X_std_test)
accuracy_score(y_test, y_svm)
roc_auc_score(y_test, y_svm)
classification_report(y_test, y_svm)
f1_score(y_test, y_svm)