Learn practical skills, build real-world projects, and advance your career
Created 3 years ago
####################
## load libraries ##
####################
import numpy as np
from sklearn.metrics import roc_auc_score
np.random.seed(1234567)
############################
## Toy Dataset Simulation ##
############################
def simulate_dataset(n=10000, features=10):
np.random.seed(1081565)
## divide number of features by 2
p = int(features/2)
## specify intercept column
X_intercept = np.ones((n,1))
## create p features over the reals
X_1 = np.random.uniform(-2, 2, size=(n, p))
## create p features with support {0,1}
X_2 = np.random.randint(low=0, high=2, size=(n, p))
## concate intercept, X1, and X2
X = np.concatenate((X_intercept, X_1, X_2), axis=1)
## save the original "Design Matrix" of features to return later
D = np.concatenate((X_1, X_2), axis=1)
## create 10 additional features
X_11 = (X[:, 1] * X[:, 1]).reshape((n, 1))
X_12 = (X[:, 3] * X[:, 3]).reshape((n, 1))
X_13 = (X[:, 2] * X[:, 4]).reshape((n, 1))
X_14 = (X[:, 2] * X[:, 8]).reshape((n, 1))
X_15 = (X[:, 7] * X[:, 9]).reshape((n, 1))
X_16 = np.cos(X[:, 4]).reshape((n, 1))
X_17 = np.sin(X[:, 2]).reshape((n, 1))
X_18 = (X_16 * X_11).reshape((n, 1))
X_19 = (X_17 * X_12).reshape((n, 1))
X_20 = (X_16 * X_16).reshape((n, 1))
X = np.concatenate((X, X_11, X_12, X_13, X_14, X_15, X_16, X_17, X_18, X_19, X_20), axis=1)
del X_intercept, X_1, X_2, X_11, X_12, X_13, X_14, X_15, X_16, X_17, X_18, X_19, X_20
## specify Betas
Beta = np.random.uniform(-1, 1, size=((2*features)+1, 1))
## Fit the outcome Y
theta = np.dot(X, Beta)
prob = (1 / (1 + np.exp(-theta)))
# ## specify categorical {0,1} outcome
# Y = np.zeros((n, 1))
# for i in range(0, n):
# Y[i, 0] = np.random.choice([0, 1], size=1, replace=True, p=[1-prob[i, 0], prob[i, 0]])
## specify categorical {0,1} outcome
Y = np.zeros((n, 1))
for i in range(0, n):
Y[i, 0] = np.random.choice([0, 1], size=1, replace=True, p=[1-prob[i, 0], prob[i, 0]])
## return outcome Y and original Design Matrix D
return(Y, D)
##############################
## Neural Network Functions ##
##############################
def Normalize_Design_Matrix(D):
D_means = D.mean(axis=0, keepdims=True)
D_std = D.std(axis=0, keepdims=True)
D_normalized = (D-D_means) / D_std
return(D_normalized, D_means, D_std)
def initialize_Beta(p1, p2=1):
Beta = np.random.uniform(-1, 1, size=(p1, p2))
return(Beta)
def initialize_b(p2=1):
b = np.random.uniform(-1, 1, size=(1, p2))
return(b)
def sigmoid(theta):
Y = 1 / (1 + np.exp(-theta))
return(Y)
def sigmoid_derivative(Z):
Y = sigmoid(Z) * (1-sigmoid(Z))
return(Y)
def softmax(theta):
Y = np.exp(theta)
Y = Y / Y.sum(axis=1).reshape((Y.shape[0],1))
return(Y)
def softmax_derivative(Z):
Y = softmax(Z) * (1-softmax(Z))
return(Y)
def tanh(theta):
Y = (np.exp(theta) - np.exp(-theta)) / (np.exp(theta) + np.exp(-theta))
return(Y)
def tanh_derivative(Z):
Y = 1 - (tanh(Z)*tanh(Z))
return(Y)
def reLu(theta):
Y = theta.copy()
Y[Y<0]=0
return(Y)
def reLu_derivative(Z):
return(1)
def identity(theta):
Y = theta.copy()
return(Y)
def identity_derivative(Z):
return(1)
def specify_mini_batch_size(n, mini_batch_size):
if(n % mini_batch_size == 0):
start = (n//mini_batch_size)*[None]
end = (n//mini_batch_size)*[None]
m = (n//mini_batch_size)*[None]
for i in range(0, (n//mini_batch_size)):
start[i] = i*mini_batch_size
end[i] = (i+1)*mini_batch_size
m[i] = mini_batch_size
del i
else:
start = int(np.ceil(n/mini_batch_size))*[None]
end = int(np.ceil(n/mini_batch_size))*[None]
m = int(np.ceil(n/mini_batch_size))*[None]
for i in range(0, (n//mini_batch_size)):
start[i] = i*mini_batch_size
end[i] = (i+1)*mini_batch_size
m[i] = mini_batch_size
start[i+1] = end[i]
end[i+1] = n
m[i+1] = end[i+1] - start[i+1]
del i
return(start, end, m)
class NeuralNetwork:
def __init__(self, Y, D):
self.n = D.shape[0]
self.Y = Y
self.D = D
self.D_normalized, self.D_means, self.D_std = Normalize_Design_Matrix(self.D)
def specify_layers(self, p, activations):
self.p = p
self.activations = activations
self.Betas = {}
self.bs = {}
self.X = {}
self.Z = {}
self.Y_hat = None
self.dJ_dZ = {}
self.dJ_dB = {}
self.dJ_db = {}
for i in range(0, len(self.p)):
if(i<len(self.p)-1):
self.Betas['B'+str(i+1)] = initialize_Beta(p1=self.p[i], p2=self.p[i+1])
self.bs['b'+str(i+1)] = initialize_b(p2=self.p[i+1])
else:
self.Betas['B'+str(i+1)] = initialize_Beta(p1=self.p[i], p2=self.Y.shape[1])
self.bs['b'+str(i+1)] = initialize_b(p2=self.Y.shape[1])
self.X['X'+str(i+1)] = None
self.dJ_dZ['dJ_dZ'+str(i+1)] = None
self.dJ_dB['dJ_dB'+str(i+1)] = None
self.dJ_db['dJ_db'+str(i+1)] = None
def forward_dropout(self, j, dropout_probability):
self.X['X'+str(j)] = (1/(1-dropout_probability)) * self.X['X'+str(j)] * np.random.binomial(1, (1-dropout_probability), size=(self.X['X'+str(j)].shape[0], self.X['X'+str(j)].shape[1]))
def forward_Z(self, j):
self.Z['Z'+str(j)] = np.dot(self.X['X'+str(j)], self.Betas['B'+str(j)]) + self.bs['b'+str(j)]
def forward_X(self, j):
activation = self.activations[j-1]
if(activation=='reLu'):
self.X['X'+str(j+1)] = reLu(self.Z['Z'+str(j)])
if(activation=='sigmoid'):
self.X['X'+str(j+1)] = sigmoid(self.Z['Z'+str(j)])
if(activation=='tanh'):
self.X['X'+str(j+1)] = tanh(self.Z['Z'+str(j)])
if(activation=='softmax'):
self.X['X'+str(j+1)] = softmax(self.Z['Z'+str(j)])
if(activation=='identity'):
self.X['X'+str(j+1)] = identity(self.Z['Z'+str(j)])
def forward_Y(self):
activation = self.activations[-1]
if(activation=='reLu'):
self.Y_hat = reLu(self.Z['Z'+str(len(self.Z))])
if(activation=='sigmoid'):
self.Y_hat = sigmoid(self.Z['Z'+str(len(self.Z))])
if(activation=='tanh'):
self.Y_hat = tanh(self.Z['Z'+str(len(self.Z))])
if(activation=='softmax'):
self.Y_hat = softmax(self.Z['Z'+str(len(self.Z))])
if(activation=='identity'):
self.Y_hat = identity(self.Z['Z'+str(len(self.Z))])
def calculate_loss_function(self, Y_mini, i, m, k):
activation = self.activations[-1]
if(activation=='sigmoid' or activation=='softmax'):
self.J = (1/m[i])*(-np.dot(Y_mini.T, np.log(self.Y_hat)) - np.dot((1-Y_mini).T, np.log(1-self.Y_hat)))
if(activation=='identity' or activation=='softmax'):
self.J = (1/m[i])*np.dot((Y_mini-self.Y_hat).T, (Y_mini-self.Y_hat))
if(i==(len(m)-1)):
print('current loss on epoch ' + str(k+1) + ' is: ' + str(self.J.sum()))
def backward_dJ_dZ(self, Y_mini, j):
activation = self.activations[j-1]
if(j==len(self.Z)):
if(activation=='sigmoid'):
self.dJ_dZ['dJ_dZ'+str(len(self.dJ_dZ))] = sigmoid(self.Z['Z'+str(len(self.Z))]) - Y_mini
if(activation=='reLu'):
self.dJ_dZ['dJ_dZ'+str(len(self.dJ_dZ))] = reLu(self.Z['Z'+str(len(self.Z))]) - Y_mini
if(activation=='tanh'):
self.dJ_dZ['dJ_dZ'+str(len(self.dJ_dZ))] = tanh(self.Z['Z'+str(len(self.Z))]) - Y_mini
if(activation=='softmax'):
self.dJ_dZ['dJ_dZ'+str(len(self.dJ_dZ))] = softmax(self.Z['Z'+str(len(self.Z))]) - Y_mini
if(activation=='identity'):
self.dJ_dZ['dJ_dZ'+str(len(self.dJ_dZ))] = identity(self.Z['Z'+str(len(self.Z))]) - Y_mini
if(j<len(self.Z)):
if(activation=='sigmoid'):
self.dJ_dZ['dJ_dZ'+str(j)] = np.dot(self.dJ_dZ['dJ_dZ'+str(j+1)], self.Betas['B'+str(j+1)].T) * sigmoid_derivative(self.Z['Z'+str(j)])
if(activation=='reLu'):
self.dJ_dZ['dJ_dZ'+str(j)] = np.dot(self.dJ_dZ['dJ_dZ'+str(j+1)], self.Betas['B'+str(j+1)].T) * reLu_derivative(self.Z['Z'+str(j)])
if(activation=='tanh'):
self.dJ_dZ['dJ_dZ'+str(j)] = np.dot(self.dJ_dZ['dJ_dZ'+str(j+1)], self.Betas['B'+str(j+1)].T) * tanh_derivative(self.Z['Z'+str(j)])
if(activation=='softmax'):
self.dJ_dZ['dJ_dZ'+str(j)] = np.dot(self.dJ_dZ['dJ_dZ'+str(j+1)], self.Betas['B'+str(j+1)].T) * softmax_derivative(self.Z['Z'+str(j)])
if(activation=='identity'):
self.dJ_dZ['dJ_dZ'+str(j)] = np.dot(self.dJ_dZ['dJ_dZ'+str(j+1)], self.Betas['B'+str(j+1)].T) * identity_derivative(self.Z['Z'+str(j)])
def backward_dJ_dB(self, j):
self.dJ_dB['dJ_dB'+str(j)] = np.dot(self.X['X'+str(j)].T, self.dJ_dZ['dJ_dZ'+str(j)])
def backward_dJ_db(self, j, i, m):
self.dJ_db['dJ_db'+str(j)] = np.dot(np.ones((1, m[i])), self.dJ_dZ['dJ_dZ'+str(j)])
def update_Betas(self, j, alpha):
self.Betas['B'+str(j)] = self.Betas['B'+str(j)] - (alpha*self.dJ_dB['dJ_dB'+str(j)])
def update_bs(self, j, alpha):
self.bs['b'+str(j)] = self.bs['b'+str(j)] - (alpha*self.dJ_db['dJ_db'+str(j)])
def fit(self, mini_batch_size, epochs=1, alpha=0.001, drop_out=False, dropout_probability=0):
###########################################################
## specify coordinates of minibatch sizes for each epoch ##
###########################################################
start, end, m = specify_mini_batch_size(self.n, mini_batch_size)
for k in range(0, epochs): ## loop over the epochs
for i in range(0,len(m)): ## loop over the minibatches
Y_mini = self.Y[start[i]:end[i],:]
###################
## forward pass: ##
###################
for j in range(1, len(self.X)):
## normalize the input layer
if(j==1):
self.X['X'+str(j)] = self.D_normalized[start[i]:end[i],:]
## initiate dropout regularization (if needed)
if(drop_out):
self.forward_dropout(j, dropout_probability)
## calculate Z of current layer
self.forward_Z(j)
## calculate X of next layer
self.forward_X(j)
## calculate Z of final layer
self.forward_Z(j+1)
## calculate output layer Y
self.forward_Y()
###########################################
## print current value of loss function: ##
###########################################
self.calculate_loss_function(Y_mini, i, m, k)
#####################
## backwards pass: ##
#####################
for j in range(len(self.Z), 0, -1):
## calculate the gradient dJ_dTheta
self.backward_dJ_dZ(Y_mini, j)
for j in range(len(self.X), 0, -1):
## calculate the gradient dJ_dB
self.backward_dJ_dB(j)
## calculate the gradient dJ_db
self.backward_dJ_db(j, i, m)
##################################
## update coefficient estimates: ##
##################################
for j in range(len(self.Betas), 0, -1):
## update Betas
self.update_Betas(j, alpha)
## update intercets b
self.update_bs(j, alpha)
del Y_mini
## simulate dataset
Y, D = simulate_dataset()
## initialize Neural Network class object
NN = NeuralNetwork(Y, D)
## fit the Neural Network
NN.specify_layers(p=[NN.D.shape[1], 100, 85, 80, 70, 60], activations=['reLu', 'sigmoid', 'reLu', 'sigmoid', 'reLu', 'sigmoid'])
NN.fit(mini_batch_size=10000, epochs=1000, alpha=0.000001, drop_out=False, dropout_probability=0.02)
current loss on epoch 1 is: 4.626386947606572
current loss on epoch 2 is: 2.643770386910068
current loss on epoch 3 is: 1.7559048935000692
current loss on epoch 4 is: 1.5914338785783473
current loss on epoch 5 is: 1.548000215292208
current loss on epoch 6 is: 1.5153298546522171
current loss on epoch 7 is: 1.48561193852702
current loss on epoch 8 is: 1.4580856182311097
current loss on epoch 9 is: 1.4324135211750917
current loss on epoch 10 is: 1.4083734564134422
current loss on epoch 11 is: 1.3857579065175314
current loss on epoch 12 is: 1.36429459841152
current loss on epoch 13 is: 1.3439364704232784
current loss on epoch 14 is: 1.324690879342278
current loss on epoch 15 is: 1.306507920300551
current loss on epoch 16 is: 1.2893254785222155
current loss on epoch 17 is: 1.273058905650284
current loss on epoch 18 is: 1.257677193746075
current loss on epoch 19 is: 1.2431193999854653
current loss on epoch 20 is: 1.2293265166495182
current loss on epoch 21 is: 1.2162722489950337
current loss on epoch 22 is: 1.2038722160906805
current loss on epoch 23 is: 1.1920896850441387
current loss on epoch 24 is: 1.1808833097818443
current loss on epoch 25 is: 1.1701812688400781
current loss on epoch 26 is: 1.1599372389770983
current loss on epoch 27 is: 1.1501519890920575
current loss on epoch 28 is: 1.1407817404586467
current loss on epoch 29 is: 1.1317857331196546
current loss on epoch 30 is: 1.1231217605174308
current loss on epoch 31 is: 1.1147742037564554
current loss on epoch 32 is: 1.1067372559083448
current loss on epoch 33 is: 1.0989612421275843
current loss on epoch 34 is: 1.091421245108141
current loss on epoch 35 is: 1.084111020732424
current loss on epoch 36 is: 1.0770117925195004
current loss on epoch 37 is: 1.070110886337951
current loss on epoch 38 is: 1.0633965748370862
current loss on epoch 39 is: 1.0568490418587642
current loss on epoch 40 is: 1.0504642947518446
current loss on epoch 41 is: 1.0442252623151025
current loss on epoch 42 is: 1.0381180201155171
current loss on epoch 43 is: 1.0321506819879014
current loss on epoch 44 is: 1.026320445936341
current loss on epoch 45 is: 1.0206172043026283
current loss on epoch 46 is: 1.015023968744693
current loss on epoch 47 is: 1.009528991505308
current loss on epoch 48 is: 1.0041258142649476
current loss on epoch 49 is: 0.9988253904366658
current loss on epoch 50 is: 0.9936124819897698
current loss on epoch 51 is: 0.9885030187841447
current loss on epoch 52 is: 0.983505543176748
current loss on epoch 53 is: 0.9786115417768001
current loss on epoch 54 is: 0.9738174151471897
current loss on epoch 55 is: 0.9691117819705611
current loss on epoch 56 is: 0.9645051710310826
current loss on epoch 57 is: 0.9599904836520351
current loss on epoch 58 is: 0.9555716128721721
current loss on epoch 59 is: 0.9512411534548796
current loss on epoch 60 is: 0.9469891359355151
current loss on epoch 61 is: 0.9428264862446318
current loss on epoch 62 is: 0.9387450315778173
current loss on epoch 63 is: 0.9347409365832204
current loss on epoch 64 is: 0.9308030821614661
current loss on epoch 65 is: 0.9269501532370833
current loss on epoch 66 is: 0.9231690271333166
current loss on epoch 67 is: 0.9194642008551485
current loss on epoch 68 is: 0.915838845555804
current loss on epoch 69 is: 0.9122960932910686
current loss on epoch 70 is: 0.9088312465332518
current loss on epoch 71 is: 0.9054339404663566
current loss on epoch 72 is: 0.9021070255693764
current loss on epoch 73 is: 0.8988408477984715
current loss on epoch 74 is: 0.895628618232483
current loss on epoch 75 is: 0.8924828104123892
current loss on epoch 76 is: 0.889399921688487
current loss on epoch 77 is: 0.886372379231814
current loss on epoch 78 is: 0.8834003545216278
current loss on epoch 79 is: 0.8804812045845393
current loss on epoch 80 is: 0.8776194039269686
current loss on epoch 81 is: 0.8748055445302205
current loss on epoch 82 is: 0.8720435401836368
current loss on epoch 83 is: 0.8693331597253515
current loss on epoch 84 is: 0.8666715718012995
current loss on epoch 85 is: 0.8640646299686134
current loss on epoch 86 is: 0.8615054012812734
current loss on epoch 87 is: 0.8589971275249576
current loss on epoch 88 is: 0.8565379281995389
current loss on epoch 89 is: 0.8541205829932886
current loss on epoch 90 is: 0.8517382734750354
current loss on epoch 91 is: 0.8493900392508053
current loss on epoch 92 is: 0.8470810136566307
current loss on epoch 93 is: 0.8448138646642117
current loss on epoch 94 is: 0.8425840983649227
current loss on epoch 95 is: 0.840397029355464
current loss on epoch 96 is: 0.838246496155003
current loss on epoch 97 is: 0.8361266329320812
current loss on epoch 98 is: 0.8340440429897372
current loss on epoch 99 is: 0.8319921529902721
current loss on epoch 100 is: 0.8299701658530112
current loss on epoch 101 is: 0.8279743809165292
current loss on epoch 102 is: 0.8260056969084072
current loss on epoch 103 is: 0.8240618238642905
current loss on epoch 104 is: 0.8221439498737296
current loss on epoch 105 is: 0.8202488357568657
current loss on epoch 106 is: 0.8183798366123901
current loss on epoch 107 is: 0.8165399500077174
current loss on epoch 108 is: 0.8147172587030723
current loss on epoch 109 is: 0.8129195299091103
current loss on epoch 110 is: 0.8111451038484728
current loss on epoch 111 is: 0.8093925189391721
current loss on epoch 112 is: 0.8076582707267841
current loss on epoch 113 is: 0.8059363428018351
current loss on epoch 114 is: 0.8042414803546518
current loss on epoch 115 is: 0.802570516867816
current loss on epoch 116 is: 0.800922674206389
current loss on epoch 117 is: 0.7992929873965543
current loss on epoch 118 is: 0.7976832346451905
current loss on epoch 119 is: 0.7960882101705408
current loss on epoch 120 is: 0.794515652882469
current loss on epoch 121 is: 0.7929677594936986
current loss on epoch 122 is: 0.7914399916810786
current loss on epoch 123 is: 0.7899300656386006
current loss on epoch 124 is: 0.788430993872257
current loss on epoch 125 is: 0.7869431676413521
current loss on epoch 126 is: 0.7854651781325025
current loss on epoch 127 is: 0.7839996616343482
current loss on epoch 128 is: 0.782550182129201
current loss on epoch 129 is: 0.7811179018446004
current loss on epoch 130 is: 0.7797032361717086
current loss on epoch 131 is: 0.7783068036656295
current loss on epoch 132 is: 0.7769271534082706
current loss on epoch 133 is: 0.7755529117555438
current loss on epoch 134 is: 0.7741900451782316
current loss on epoch 135 is: 0.7728422853475578
current loss on epoch 136 is: 0.7715100213781045
current loss on epoch 137 is: 0.7701920391481356
current loss on epoch 138 is: 0.7688878323328247
current loss on epoch 139 is: 0.7675967440421204
current loss on epoch 140 is: 0.7663180867410062
current loss on epoch 141 is: 0.765050985357848
current loss on epoch 142 is: 0.7638025051277556
current loss on epoch 143 is: 0.7625705423269943
current loss on epoch 144 is: 0.7613547630229517
current loss on epoch 145 is: 0.760149941736322
current loss on epoch 146 is: 0.7589558687018187
current loss on epoch 147 is: 0.7577686462081105
current loss on epoch 148 is: 0.7565898900873764
current loss on epoch 149 is: 0.7554188849546665
current loss on epoch 150 is: 0.7542592799036584
current loss on epoch 151 is: 0.7531085136586179
current loss on epoch 152 is: 0.751968228175898
current loss on epoch 153 is: 0.7508406291025623
current loss on epoch 154 is: 0.7497215155786885
current loss on epoch 155 is: 0.7486051562053284
current loss on epoch 156 is: 0.7474981623244666
current loss on epoch 157 is: 0.7464002574154144
current loss on epoch 158 is: 0.7453127926450791
current loss on epoch 159 is: 0.7442380476279222
current loss on epoch 160 is: 0.7431666527939476
current loss on epoch 161 is: 0.7421042386001205
current loss on epoch 162 is: 0.7410503113509295
current loss on epoch 163 is: 0.7400056455946474
current loss on epoch 164 is: 0.7389648118909414
current loss on epoch 165 is: 0.7379303396856128
current loss on epoch 166 is: 0.7369041696256348
current loss on epoch 167 is: 0.7358850242641864
current loss on epoch 168 is: 0.7348734546586193
current loss on epoch 169 is: 0.7338724432724023
current loss on epoch 170 is: 0.7328839876334559
current loss on epoch 171 is: 0.7319045986457993
current loss on epoch 172 is: 0.7309353474806641
current loss on epoch 173 is: 0.7299777771868312
current loss on epoch 174 is: 0.7290285775604898
current loss on epoch 175 is: 0.7280863649699785
current loss on epoch 176 is: 0.7271510144412136
current loss on epoch 177 is: 0.7262245964367203
current loss on epoch 178 is: 0.725305668919695
current loss on epoch 179 is: 0.7243959835707796
current loss on epoch 180 is: 0.7234956963743926
current loss on epoch 181 is: 0.7226038863996064
current loss on epoch 182 is: 0.7217185617112447
current loss on epoch 183 is: 0.7208408473289543
current loss on epoch 184 is: 0.7199674181945331
current loss on epoch 185 is: 0.719102544412258
current loss on epoch 186 is: 0.7182463842286041
current loss on epoch 187 is: 0.717396001219208
current loss on epoch 188 is: 0.7165516383139562
current loss on epoch 189 is: 0.71571148758733
current loss on epoch 190 is: 0.7148780687196837
current loss on epoch 191 is: 0.7140543375034272
current loss on epoch 192 is: 0.7132355673082781
current loss on epoch 193 is: 0.7124223084984389
current loss on epoch 194 is: 0.7116154733689272
current loss on epoch 195 is: 0.7108148766728963
current loss on epoch 196 is: 0.7100213627952612
current loss on epoch 197 is: 0.7092327588527942
current loss on epoch 198 is: 0.7084505757617658
current loss on epoch 199 is: 0.7076748706141986
current loss on epoch 200 is: 0.7069062130046919
current loss on epoch 201 is: 0.7061437651220611
current loss on epoch 202 is: 0.7053875773791634
current loss on epoch 203 is: 0.7046378494229047
current loss on epoch 204 is: 0.7038924593046074
current loss on epoch 205 is: 0.7031501359930067
current loss on epoch 206 is: 0.7024133291039816
current loss on epoch 207 is: 0.7016840228301353
current loss on epoch 208 is: 0.7009619404755251
current loss on epoch 209 is: 0.7002481120374544
current loss on epoch 210 is: 0.6995390504906461
current loss on epoch 211 is: 0.698832826840106
current loss on epoch 212 is: 0.6981297900031617
current loss on epoch 213 is: 0.6974346932866662
current loss on epoch 214 is: 0.6967462699538388
current loss on epoch 215 is: 0.6960632416747429
current loss on epoch 216 is: 0.6953831031312098
current loss on epoch 217 is: 0.6947077693537593
current loss on epoch 218 is: 0.6940374136745524
current loss on epoch 219 is: 0.6933769304775355
current loss on epoch 220 is: 0.6927222141912688
current loss on epoch 221 is: 0.6920724726775498
current loss on epoch 222 is: 0.6914260340227764
current loss on epoch 223 is: 0.6907834689495953
current loss on epoch 224 is: 0.6901466532846786
current loss on epoch 225 is: 0.6895156045067439
current loss on epoch 226 is: 0.6888919066427139
current loss on epoch 227 is: 0.6882753014821298
current loss on epoch 228 is: 0.6876623641369223
current loss on epoch 229 is: 0.6870530555177994
current loss on epoch 230 is: 0.6864477487438272
current loss on epoch 231 is: 0.6858462356655093
current loss on epoch 232 is: 0.6852506548162123
current loss on epoch 233 is: 0.6846600586027811
current loss on epoch 234 is: 0.6840750574248626
current loss on epoch 235 is: 0.6834949841089145
current loss on epoch 236 is: 0.6829198743647148
current loss on epoch 237 is: 0.6823488822425697
current loss on epoch 238 is: 0.6817796138737493
current loss on epoch 239 is: 0.6812124598544677
current loss on epoch 240 is: 0.6806475218095447
current loss on epoch 241 is: 0.6800836442322209
current loss on epoch 242 is: 0.6795239930969575
current loss on epoch 243 is: 0.6789692686426745
current loss on epoch 244 is: 0.6784188838933027
current loss on epoch 245 is: 0.6778725656471752
current loss on epoch 246 is: 0.6773296117641413
current loss on epoch 247 is: 0.6767897815440928
current loss on epoch 248 is: 0.6762532473892403
current loss on epoch 249 is: 0.6757198989371173
current loss on epoch 250 is: 0.675192164873625
current loss on epoch 251 is: 0.674671969778995
current loss on epoch 252 is: 0.6741589595736496
current loss on epoch 253 is: 0.673649006290945
current loss on epoch 254 is: 0.6731412961149439
current loss on epoch 255 is: 0.6726368505615263
current loss on epoch 256 is: 0.672136361294133
current loss on epoch 257 is: 0.671638254034561
current loss on epoch 258 is: 0.6711437730687463
current loss on epoch 259 is: 0.6706512476280543
current loss on epoch 260 is: 0.6701614952652286
current loss on epoch 261 is: 0.6696738527933551
current loss on epoch 262 is: 0.6691894344382223
current loss on epoch 263 is: 0.6687070004358964
current loss on epoch 264 is: 0.6682261854931106
current loss on epoch 265 is: 0.6677471622837442
current loss on epoch 266 is: 0.667268755469354
current loss on epoch 267 is: 0.6667940766760915
current loss on epoch 268 is: 0.6663229801987196
current loss on epoch 269 is: 0.6658559401313758
current loss on epoch 270 is: 0.6653919261414705
current loss on epoch 271 is: 0.6649293798900372
current loss on epoch 272 is: 0.6644686413811615
current loss on epoch 273 is: 0.6640095655286881
current loss on epoch 274 is: 0.6635506551973371
current loss on epoch 275 is: 0.6630926360337326
current loss on epoch 276 is: 0.662635816261162
current loss on epoch 277 is: 0.6621808934491729
current loss on epoch 278 is: 0.6617299906219791
current loss on epoch 279 is: 0.6612801452604243
current loss on epoch 280 is: 0.6608320539237742
current loss on epoch 281 is: 0.6603851331634897
current loss on epoch 282 is: 0.6599397647005801
current loss on epoch 283 is: 0.6594976592074919
current loss on epoch 284 is: 0.65905776946027
current loss on epoch 285 is: 0.6586209143877301
current loss on epoch 286 is: 0.6581855688404636
current loss on epoch 287 is: 0.6577530556478557
current loss on epoch 288 is: 0.6573228476703314
current loss on epoch 289 is: 0.6568956684640839
current loss on epoch 290 is: 0.6564710836682963
current loss on epoch 291 is: 0.6560492622866301
current loss on epoch 292 is: 0.6556297388560054
current loss on epoch 293 is: 0.6552116398797821
current loss on epoch 294 is: 0.6547958648990424
current loss on epoch 295 is: 0.6543817235782197
current loss on epoch 296 is: 0.6539696544977478
current loss on epoch 297 is: 0.6535602095436451
current loss on epoch 298 is: 0.653153400004329
current loss on epoch 299 is: 0.6527500261468484
current loss on epoch 300 is: 0.6523492660045616
current loss on epoch 301 is: 0.651952429023762
current loss on epoch 302 is: 0.6515580919563382
current loss on epoch 303 is: 0.6511654697836845
current loss on epoch 304 is: 0.6507738065742279
current loss on epoch 305 is: 0.6503836574327517
current loss on epoch 306 is: 0.6499942262441569
current loss on epoch 307 is: 0.6496060981630072
current loss on epoch 308 is: 0.6492187618549432
current loss on epoch 309 is: 0.6488326167295805
current loss on epoch 310 is: 0.6484485523372816
current loss on epoch 311 is: 0.6480674514632591
current loss on epoch 312 is: 0.6476902626655795
current loss on epoch 313 is: 0.6473161739193769
current loss on epoch 314 is: 0.6469450632575333
current loss on epoch 315 is: 0.6465772768254614
current loss on epoch 316 is: 0.6462117207149133
current loss on epoch 317 is: 0.645848224786848
current loss on epoch 318 is: 0.6454868348856985
current loss on epoch 319 is: 0.6451270903292885
current loss on epoch 320 is: 0.6447695510039426
current loss on epoch 321 is: 0.6444123117258335
current loss on epoch 322 is: 0.6440571149192338
current loss on epoch 323 is: 0.6437046889002727
current loss on epoch 324 is: 0.6433539409137798
current loss on epoch 325 is: 0.6430051923702506
current loss on epoch 326 is: 0.642657925613793
current loss on epoch 327 is: 0.6423121096816564
current loss on epoch 328 is: 0.641967270390802
current loss on epoch 329 is: 0.6416234793313358
current loss on epoch 330 is: 0.6412815384572392
current loss on epoch 331 is: 0.6409420693649478
current loss on epoch 332 is: 0.6406039179834349
current loss on epoch 333 is: 0.6402669362300982
current loss on epoch 334 is: 0.6399307695419418
current loss on epoch 335 is: 0.6395955409591141
current loss on epoch 336 is: 0.6392621786632852
current loss on epoch 337 is: 0.638931851551886
current loss on epoch 338 is: 0.6386040451660465
current loss on epoch 339 is: 0.6382762163544926
current loss on epoch 340 is: 0.6379490337951794
current loss on epoch 341 is: 0.6376236737394302
current loss on epoch 342 is: 0.6372985377210529
current loss on epoch 343 is: 0.6369750302630067
current loss on epoch 344 is: 0.6366522838675115
current loss on epoch 345 is: 0.6363302795656557
current loss on epoch 346 is: 0.6360103545598049
current loss on epoch 347 is: 0.6356928339745902
current loss on epoch 348 is: 0.6353758818503993
current loss on epoch 349 is: 0.6350608256196655
current loss on epoch 350 is: 0.6347484526921029
current loss on epoch 351 is: 0.6344379203148586
current loss on epoch 352 is: 0.634129475364104
current loss on epoch 353 is: 0.6338223624342422
current loss on epoch 354 is: 0.6335165242568773
current loss on epoch 355 is: 0.6332117819606065
current loss on epoch 356 is: 0.6329084215261732
current loss on epoch 357 is: 0.632605875295375
current loss on epoch 358 is: 0.6323045462375314
current loss on epoch 359 is: 0.6320036485831687
current loss on epoch 360 is: 0.6317038332975247
current loss on epoch 361 is: 0.631404177488971
current loss on epoch 362 is: 0.6311057390058045
current loss on epoch 363 is: 0.6308071352826868
current loss on epoch 364 is: 0.6305087474417597
current loss on epoch 365 is: 0.6302119606818364
current loss on epoch 366 is: 0.6299172502186268
current loss on epoch 367 is: 0.6296236359319872
current loss on epoch 368 is: 0.6293317133197414
current loss on epoch 369 is: 0.6290413062942971
current loss on epoch 370 is: 0.6287520297821546
current loss on epoch 371 is: 0.6284632611595524
current loss on epoch 372 is: 0.6281747285794097
current loss on epoch 373 is: 0.6278879524330021
current loss on epoch 374 is: 0.6276040507356567
current loss on epoch 375 is: 0.6273215221489234
current loss on epoch 376 is: 0.6270399424585082
current loss on epoch 377 is: 0.6267607699719702
current loss on epoch 378 is: 0.6264847351979619
current loss on epoch 379 is: 0.6262096216641024
current loss on epoch 380 is: 0.6259359237560597
current loss on epoch 381 is: 0.6256620852431733
current loss on epoch 382 is: 0.625388706817135
current loss on epoch 383 is: 0.625117218303445
current loss on epoch 384 is: 0.6248457496533736
current loss on epoch 385 is: 0.6245747779061908
current loss on epoch 386 is: 0.6243032911865691
current loss on epoch 387 is: 0.6240320012198431
current loss on epoch 388 is: 0.6237629336818186
current loss on epoch 389 is: 0.6234945945746133
current loss on epoch 390 is: 0.6232270817200083
current loss on epoch 391 is: 0.6229606267067462
current loss on epoch 392 is: 0.6226958522088115
current loss on epoch 393 is: 0.6224328532792114
current loss on epoch 394 is: 0.6221713858420501
current loss on epoch 395 is: 0.6219102713942166
current loss on epoch 396 is: 0.6216491812792432
current loss on epoch 397 is: 0.6213889210501667
current loss on epoch 398 is: 0.6211301800967398
current loss on epoch 399 is: 0.6208725368700604
current loss on epoch 400 is: 0.6206149431715486
current loss on epoch 401 is: 0.6203575364436757
current loss on epoch 402 is: 0.6201011793418338
current loss on epoch 403 is: 0.619846136561996
current loss on epoch 404 is: 0.6195923589262876
current loss on epoch 405 is: 0.6193384385499631
current loss on epoch 406 is: 0.6190856487504752
current loss on epoch 407 is: 0.6188335797236245
current loss on epoch 408 is: 0.6185823682423546
current loss on epoch 409 is: 0.6183322371677183
current loss on epoch 410 is: 0.6180833319735384
current loss on epoch 411 is: 0.6178357565780289
current loss on epoch 412 is: 0.6175895703103715
current loss on epoch 413 is: 0.6173439084480237
current loss on epoch 414 is: 0.6170994198584329
current loss on epoch 415 is: 0.6168552909713311
current loss on epoch 416 is: 0.6166120724735988
current loss on epoch 417 is: 0.6163685932149716
current loss on epoch 418 is: 0.6161260004613529
current loss on epoch 419 is: 0.6158852347963636
current loss on epoch 420 is: 0.615646246645399
current loss on epoch 421 is: 0.6154081084182801
current loss on epoch 422 is: 0.6151708487659612
current loss on epoch 423 is: 0.6149345798500905
current loss on epoch 424 is: 0.6147012696956526
current loss on epoch 425 is: 0.6144689531941135
current loss on epoch 426 is: 0.6142370981538583
current loss on epoch 427 is: 0.6140053158053311
current loss on epoch 428 is: 0.6137733625009901
current loss on epoch 429 is: 0.6135415994800347
current loss on epoch 430 is: 0.6133110209948497
current loss on epoch 431 is: 0.6130821890651181
current loss on epoch 432 is: 0.6128545450191816
current loss on epoch 433 is: 0.6126282520823889
current loss on epoch 434 is: 0.6124033041602649
current loss on epoch 435 is: 0.6121799147862567
current loss on epoch 436 is: 0.6119572893560855
current loss on epoch 437 is: 0.6117347829195323
current loss on epoch 438 is: 0.6115131668973254
current loss on epoch 439 is: 0.6112923870795283
current loss on epoch 440 is: 0.6110735343261251
current loss on epoch 441 is: 0.6108564932864232
current loss on epoch 442 is: 0.6106396550126924
current loss on epoch 443 is: 0.6104229332365673
current loss on epoch 444 is: 0.6102059654388422
current loss on epoch 445 is: 0.609990186891934
current loss on epoch 446 is: 0.6097754305039929
current loss on epoch 447 is: 0.6095624702939615
current loss on epoch 448 is: 0.6093508451304719
current loss on epoch 449 is: 0.6091397853094114
current loss on epoch 450 is: 0.6089293448086921
current loss on epoch 451 is: 0.6087200605243843
current loss on epoch 452 is: 0.6085115066033708
current loss on epoch 453 is: 0.6083033181375785
current loss on epoch 454 is: 0.6080950353258618
current loss on epoch 455 is: 0.6078881261768553
current loss on epoch 456 is: 0.6076818416701428
current loss on epoch 457 is: 0.6074757990667878
current loss on epoch 458 is: 0.6072698241698586
current loss on epoch 459 is: 0.6070647803792838
current loss on epoch 460 is: 0.6068605758603877
current loss on epoch 461 is: 0.6066574819226724
current loss on epoch 462 is: 0.6064552865251811
current loss on epoch 463 is: 0.6062548236681363
current loss on epoch 464 is: 0.6060550892180747
current loss on epoch 465 is: 0.6058562365193552
current loss on epoch 466 is: 0.6056582730630451
current loss on epoch 467 is: 0.6054610942812376
current loss on epoch 468 is: 0.6052655082993262
current loss on epoch 469 is: 0.6050704211366961
current loss on epoch 470 is: 0.6048757431510468
current loss on epoch 471 is: 0.6046827370523189
current loss on epoch 472 is: 0.6044915087478496
current loss on epoch 473 is: 0.6043020864758609
current loss on epoch 474 is: 0.6041133036493906
current loss on epoch 475 is: 0.6039256333505822
current loss on epoch 476 is: 0.6037387726595884
current loss on epoch 477 is: 0.6035517858713255
current loss on epoch 478 is: 0.6033648692270462
current loss on epoch 479 is: 0.6031781755888244
current loss on epoch 480 is: 0.6029929910571169
current loss on epoch 481 is: 0.6028098158402542
current loss on epoch 482 is: 0.6026277951184719
current loss on epoch 483 is: 0.6024468765801759
current loss on epoch 484 is: 0.6022658522308777
current loss on epoch 485 is: 0.6020848635962632
current loss on epoch 486 is: 0.6019049832697173
current loss on epoch 487 is: 0.6017259857803867
current loss on epoch 488 is: 0.6015476230196212
current loss on epoch 489 is: 0.6013695878173296
current loss on epoch 490 is: 0.6011924730167288
current loss on epoch 491 is: 0.6010153750204016
current loss on epoch 492 is: 0.600839104121714
current loss on epoch 493 is: 0.6006632581096165
current loss on epoch 494 is: 0.6004889308396076
current loss on epoch 495 is: 0.6003157931391347
current loss on epoch 496 is: 0.6001426016831969
current loss on epoch 497 is: 0.5999698252375358
current loss on epoch 498 is: 0.5997981050530702
current loss on epoch 499 is: 0.599625471819395
current loss on epoch 500 is: 0.5994533598246937
current loss on epoch 501 is: 0.599282137995584
current loss on epoch 502 is: 0.5991116038427144
current loss on epoch 503 is: 0.5989412220604673
current loss on epoch 504 is: 0.5987712902631582
current loss on epoch 505 is: 0.5986028988303277
current loss on epoch 506 is: 0.5984352543516099
current loss on epoch 507 is: 0.5982674118612257
current loss on epoch 508 is: 0.5981008328122119
current loss on epoch 509 is: 0.5979352903860717
current loss on epoch 510 is: 0.5977702488288232
current loss on epoch 511 is: 0.5976058240177239
current loss on epoch 512 is: 0.5974422955900252
current loss on epoch 513 is: 0.5972786161013853
current loss on epoch 514 is: 0.5971155506768506
current loss on epoch 515 is: 0.5969534346129752
current loss on epoch 516 is: 0.5967923476250533
current loss on epoch 517 is: 0.5966326373462701
current loss on epoch 518 is: 0.5964736148892674
current loss on epoch 519 is: 0.5963156115227479
current loss on epoch 520 is: 0.5961579626811498
current loss on epoch 521 is: 0.5959999364244719
current loss on epoch 522 is: 0.5958420131085002
current loss on epoch 523 is: 0.5956844277550355
current loss on epoch 524 is: 0.5955267954357308
current loss on epoch 525 is: 0.5953700526862972
current loss on epoch 526 is: 0.595214156485036
current loss on epoch 527 is: 0.5950579596311291
current loss on epoch 528 is: 0.5949012228509671
current loss on epoch 529 is: 0.594744857619278
current loss on epoch 530 is: 0.5945895274694635
current loss on epoch 531 is: 0.5944361034677975
current loss on epoch 532 is: 0.5942836433903581
current loss on epoch 533 is: 0.5941318903569826
current loss on epoch 534 is: 0.5939813253153277
current loss on epoch 535 is: 0.5938319414930193
current loss on epoch 536 is: 0.5936817972503806
current loss on epoch 537 is: 0.593531665763248
current loss on epoch 538 is: 0.5933820624129151
current loss on epoch 539 is: 0.5932334080776454
current loss on epoch 540 is: 0.5930863086757819
current loss on epoch 541 is: 0.5929403752949532
current loss on epoch 542 is: 0.5927946255176603
current loss on epoch 543 is: 0.5926495812518877
current loss on epoch 544 is: 0.5925055230232007
current loss on epoch 545 is: 0.5923621955395217
current loss on epoch 546 is: 0.5922183087242274
current loss on epoch 547 is: 0.5920743221406187
current loss on epoch 548 is: 0.5919312203237999
current loss on epoch 549 is: 0.5917878130255425
current loss on epoch 550 is: 0.5916452838196952
current loss on epoch 551 is: 0.5915033948973729
current loss on epoch 552 is: 0.5913632524028289
current loss on epoch 553 is: 0.5912244465336081
current loss on epoch 554 is: 0.5910868162002815
current loss on epoch 555 is: 0.5909495142039481
current loss on epoch 556 is: 0.5908128797522675
current loss on epoch 557 is: 0.5906769722560328
current loss on epoch 558 is: 0.5905408659457727
current loss on epoch 559 is: 0.590404907065145
current loss on epoch 560 is: 0.5902699374201913
current loss on epoch 561 is: 0.5901359966470454
current loss on epoch 562 is: 0.5900015265336783
current loss on epoch 563 is: 0.5898678243006193
current loss on epoch 564 is: 0.5897348898204618
current loss on epoch 565 is: 0.5896025863495923
current loss on epoch 566 is: 0.5894706858730054
current loss on epoch 567 is: 0.5893393452421483
current loss on epoch 568 is: 0.589209015460797
current loss on epoch 569 is: 0.589079320895281
current loss on epoch 570 is: 0.5889501099679928
current loss on epoch 571 is: 0.58882106645265
current loss on epoch 572 is: 0.5886929716882031
current loss on epoch 573 is: 0.5885654265416607
current loss on epoch 574 is: 0.5884380357526244
current loss on epoch 575 is: 0.5883102362030393
current loss on epoch 576 is: 0.5881829533746148
current loss on epoch 577 is: 0.5880561156361644
current loss on epoch 578 is: 0.5879299176418745
current loss on epoch 579 is: 0.5878047426261036
current loss on epoch 580 is: 0.5876792311156179
current loss on epoch 581 is: 0.587553943832113
current loss on epoch 582 is: 0.5874287359703869
current loss on epoch 583 is: 0.5873037444850637
current loss on epoch 584 is: 0.5871796475462682
current loss on epoch 585 is: 0.5870563875925875
current loss on epoch 586 is: 0.586933493646758
current loss on epoch 587 is: 0.5868109853329148
current loss on epoch 588 is: 0.5866883087931463
current loss on epoch 589 is: 0.5865661639764234
current loss on epoch 590 is: 0.5864439480822212
current loss on epoch 591 is: 0.5863223144882063
current loss on epoch 592 is: 0.5862006116616838
current loss on epoch 593 is: 0.5860793228250039
current loss on epoch 594 is: 0.5859588395923997
current loss on epoch 595 is: 0.5858387690297641
current loss on epoch 596 is: 0.5857190243768726
current loss on epoch 597 is: 0.585599779016316
current loss on epoch 598 is: 0.5854814672706125
current loss on epoch 599 is: 0.5853641895555988
current loss on epoch 600 is: 0.5852475274588419
current loss on epoch 601 is: 0.5851311938560956
current loss on epoch 602 is: 0.5850147414938809
current loss on epoch 603 is: 0.5848983266238941
current loss on epoch 604 is: 0.5847822428818998
current loss on epoch 605 is: 0.5846667835478067
current loss on epoch 606 is: 0.5845513791456665
current loss on epoch 607 is: 0.584435962038957
current loss on epoch 608 is: 0.5843205852979158
current loss on epoch 609 is: 0.5842051107584629
current loss on epoch 610 is: 0.5840898071025123
current loss on epoch 611 is: 0.583975011172221
current loss on epoch 612 is: 0.5838604165110475
current loss on epoch 613 is: 0.5837460586552646
current loss on epoch 614 is: 0.5836321162660268
current loss on epoch 615 is: 0.5835190583552358
current loss on epoch 616 is: 0.5834053557842487
current loss on epoch 617 is: 0.5832919947584516
current loss on epoch 618 is: 0.583178959638666
current loss on epoch 619 is: 0.5830662865431354
current loss on epoch 620 is: 0.5829540206058536
current loss on epoch 621 is: 0.5828418518409667
current loss on epoch 622 is: 0.5827302678147604
current loss on epoch 623 is: 0.5826191779966293
current loss on epoch 624 is: 0.5825088948436447
current loss on epoch 625 is: 0.5823991721230598
current loss on epoch 626 is: 0.5822902434754595
current loss on epoch 627 is: 0.5821819698814563
current loss on epoch 628 is: 0.5820732773319706
current loss on epoch 629 is: 0.5819641576009079
current loss on epoch 630 is: 0.5818552175386915
current loss on epoch 631 is: 0.5817468152934616
current loss on epoch 632 is: 0.5816390088189541
current loss on epoch 633 is: 0.581531449326272
current loss on epoch 634 is: 0.5814239711910391
current loss on epoch 635 is: 0.581316339088743
current loss on epoch 636 is: 0.5812082894513002
current loss on epoch 637 is: 0.5811010729363357
current loss on epoch 638 is: 0.5809944918168062
current loss on epoch 639 is: 0.5808880178002915
current loss on epoch 640 is: 0.5807821612530566
current loss on epoch 641 is: 0.5806764223118351
current loss on epoch 642 is: 0.5805711579164273
current loss on epoch 643 is: 0.580466351407943
current loss on epoch 644 is: 0.5803623177258909
current loss on epoch 645 is: 0.580258656814762
current loss on epoch 646 is: 0.5801551102080171
current loss on epoch 647 is: 0.5800517524031388
current loss on epoch 648 is: 0.5799488937458043
current loss on epoch 649 is: 0.5798459576180444
current loss on epoch 650 is: 0.579743433906924
current loss on epoch 651 is: 0.5796418782422634
current loss on epoch 652 is: 0.5795404370168311
current loss on epoch 653 is: 0.5794395540171553
current loss on epoch 654 is: 0.5793387050216927
current loss on epoch 655 is: 0.5792383822648335
current loss on epoch 656 is: 0.5791382504718701
current loss on epoch 657 is: 0.5790377406784438
current loss on epoch 658 is: 0.5789369712773588
current loss on epoch 659 is: 0.5788362351749314
current loss on epoch 660 is: 0.578735835617956
current loss on epoch 661 is: 0.5786355823764517
current loss on epoch 662 is: 0.578535849676755
current loss on epoch 663 is: 0.5784365237490209
current loss on epoch 664 is: 0.5783376233410965
current loss on epoch 665 is: 0.5782392337911524
current loss on epoch 666 is: 0.5781416376710109
current loss on epoch 667 is: 0.5780444411423
current loss on epoch 668 is: 0.5779474329462995
current loss on epoch 669 is: 0.5778505959707858
current loss on epoch 670 is: 0.5777543014988517
current loss on epoch 671 is: 0.5776592328618138
current loss on epoch 672 is: 0.5775648130933563
current loss on epoch 673 is: 0.5774700391730307
current loss on epoch 674 is: 0.5773756264295223
current loss on epoch 675 is: 0.5772816587832457
current loss on epoch 676 is: 0.5771878732586992
current loss on epoch 677 is: 0.577094158155298
current loss on epoch 678 is: 0.5770005814668162
current loss on epoch 679 is: 0.5769074142782499
current loss on epoch 680 is: 0.5768140728616684
current loss on epoch 681 is: 0.5767203882913581
current loss on epoch 682 is: 0.5766270559583129
current loss on epoch 683 is: 0.576533930977966
current loss on epoch 684 is: 0.5764412139451051
current loss on epoch 685 is: 0.5763483311618735
current loss on epoch 686 is: 0.5762553213446219
current loss on epoch 687 is: 0.5761625479231589
current loss on epoch 688 is: 0.5760695197176791
current loss on epoch 689 is: 0.5759758771569016
current loss on epoch 690 is: 0.5758819329811077
current loss on epoch 691 is: 0.5757874965144282
current loss on epoch 692 is: 0.5756938390747952
current loss on epoch 693 is: 0.5755998656824054
current loss on epoch 694 is: 0.575505587495649
current loss on epoch 695 is: 0.5754110093015051
current loss on epoch 696 is: 0.5753161803109879
current loss on epoch 697 is: 0.5752215975115731
current loss on epoch 698 is: 0.5751263161361021
current loss on epoch 699 is: 0.5750313910974548
current loss on epoch 700 is: 0.5749371128467563
current loss on epoch 701 is: 0.574843520609205
current loss on epoch 702 is: 0.5747502215398917
current loss on epoch 703 is: 0.574657874970744
current loss on epoch 704 is: 0.5745659778482219
current loss on epoch 705 is: 0.5744745916775783
current loss on epoch 706 is: 0.5743833594421387
current loss on epoch 707 is: 0.5742925820957666
current loss on epoch 708 is: 0.5742024005028867
current loss on epoch 709 is: 0.5741123011845943
current loss on epoch 710 is: 0.5740232574200853
current loss on epoch 711 is: 0.573934938248335
current loss on epoch 712 is: 0.5738467698816214
current loss on epoch 713 is: 0.5737580428897125
current loss on epoch 714 is: 0.5736691890798821
current loss on epoch 715 is: 0.5735800057243933
current loss on epoch 716 is: 0.5734919399157636
current loss on epoch 717 is: 0.5734045732665681
current loss on epoch 718 is: 0.5733181968361887
current loss on epoch 719 is: 0.5732318194276479
current loss on epoch 720 is: 0.5731450198647934
current loss on epoch 721 is: 0.5730583033155674
current loss on epoch 722 is: 0.5729717552820256
current loss on epoch 723 is: 0.5728858639980725
current loss on epoch 724 is: 0.5727996946475307
current loss on epoch 725 is: 0.5727133853846882
current loss on epoch 726 is: 0.5726272094007442
current loss on epoch 727 is: 0.5725416206429856
current loss on epoch 728 is: 0.5724563454821215
current loss on epoch 729 is: 0.5723713662349669
current loss on epoch 730 is: 0.5722866191333851
current loss on epoch 731 is: 0.5722021747576294
current loss on epoch 732 is: 0.5721175159317403
current loss on epoch 733 is: 0.5720325566330009
current loss on epoch 734 is: 0.5719479305575634
current loss on epoch 735 is: 0.5718638604798546
current loss on epoch 736 is: 0.5717798120415212
current loss on epoch 737 is: 0.5716955955484375
current loss on epoch 738 is: 0.5716119776516488
current loss on epoch 739 is: 0.57152932610604
current loss on epoch 740 is: 0.5714471860864744
current loss on epoch 741 is: 0.5713648483224947
current loss on epoch 742 is: 0.5712826823366971
current loss on epoch 743 is: 0.5712008670463482
current loss on epoch 744 is: 0.5711196156941631
current loss on epoch 745 is: 0.5710384334247023
current loss on epoch 746 is: 0.5709574793209965
current loss on epoch 747 is: 0.5708766576665815
current loss on epoch 748 is: 0.5707962619176434
current loss on epoch 749 is: 0.5707167028211079
current loss on epoch 750 is: 0.5706376357299523
current loss on epoch 751 is: 0.5705591530242978
current loss on epoch 752 is: 0.5704814241742896
current loss on epoch 753 is: 0.5704040893641035
current loss on epoch 754 is: 0.5703268768059502
current loss on epoch 755 is: 0.5702498549173249
current loss on epoch 756 is: 0.5701738058010244
current loss on epoch 757 is: 0.5700986304215536
current loss on epoch 758 is: 0.5700236478134876
current loss on epoch 759 is: 0.5699488197422711
current loss on epoch 760 is: 0.5698747264575649
current loss on epoch 761 is: 0.569800267867787
current loss on epoch 762 is: 0.5697256015425539
current loss on epoch 763 is: 0.5696513327128123
current loss on epoch 764 is: 0.5695769219584603
current loss on epoch 765 is: 0.5695027745151378
current loss on epoch 766 is: 0.5694287411005245
current loss on epoch 767 is: 0.569354841706054
current loss on epoch 768 is: 0.5692814435012618
current loss on epoch 769 is: 0.5692088389057901
current loss on epoch 770 is: 0.5691364748804055
current loss on epoch 771 is: 0.5690647449546639
current loss on epoch 772 is: 0.5689933344772466
current loss on epoch 773 is: 0.5689222979010435
current loss on epoch 774 is: 0.5688515417058805
current loss on epoch 775 is: 0.5687807760928403
current loss on epoch 776 is: 0.5687102888245772
current loss on epoch 777 is: 0.5686399781121393
current loss on epoch 778 is: 0.5685694499044667
current loss on epoch 779 is: 0.5684975768265101
current loss on epoch 780 is: 0.5684254992731796
current loss on epoch 781 is: 0.5683533209562913
current loss on epoch 782 is: 0.5682810102622263
current loss on epoch 783 is: 0.5682087563181851
current loss on epoch 784 is: 0.5681370327766431
current loss on epoch 785 is: 0.5680662767190788
current loss on epoch 786 is: 0.567996228735456
current loss on epoch 787 is: 0.5679268022086152
current loss on epoch 788 is: 0.5678577770794858
current loss on epoch 789 is: 0.567789435745765
current loss on epoch 790 is: 0.5677206548593327
current loss on epoch 791 is: 0.5676513388335747
current loss on epoch 792 is: 0.5675820256369539
current loss on epoch 793 is: 0.5675127269347058
current loss on epoch 794 is: 0.5674434484428592
current loss on epoch 795 is: 0.5673735912747796
current loss on epoch 796 is: 0.5673037348298001
current loss on epoch 797 is: 0.5672335286791124
current loss on epoch 798 is: 0.5671640799945035
current loss on epoch 799 is: 0.5670948986876354
current loss on epoch 800 is: 0.5670259282180292
current loss on epoch 801 is: 0.5669570550203437
current loss on epoch 802 is: 0.5668883180314364
current loss on epoch 803 is: 0.5668193857135396
current loss on epoch 804 is: 0.5667501968249371
current loss on epoch 805 is: 0.5666805868999365
current loss on epoch 806 is: 0.5666107924077763
current loss on epoch 807 is: 0.566541544487038
current loss on epoch 808 is: 0.5664728462592411
current loss on epoch 809 is: 0.5664041947337155
current loss on epoch 810 is: 0.5663355927268087
current loss on epoch 811 is: 0.56626749278169
current loss on epoch 812 is: 0.5662000212937967
current loss on epoch 813 is: 0.5661326068569908
current loss on epoch 814 is: 0.5660648149456967
current loss on epoch 815 is: 0.5659970259616096
current loss on epoch 816 is: 0.5659292211433833
current loss on epoch 817 is: 0.5658615160204395
current loss on epoch 818 is: 0.5657940087924008
current loss on epoch 819 is: 0.5657263402958389
current loss on epoch 820 is: 0.5656590590866047
current loss on epoch 821 is: 0.565591932860124
current loss on epoch 822 is: 0.5655248681367026
current loss on epoch 823 is: 0.5654578487882361
current loss on epoch 824 is: 0.56539059229833
current loss on epoch 825 is: 0.5653236080702918
current loss on epoch 826 is: 0.5652566409196472
current loss on epoch 827 is: 0.5651899159851008
current loss on epoch 828 is: 0.5651235983405856
current loss on epoch 829 is: 0.5650586467639868
current loss on epoch 830 is: 0.5649940705001772
current loss on epoch 831 is: 0.5649294342706296
current loss on epoch 832 is: 0.564864737816744
current loss on epoch 833 is: 0.5648008883369586
current loss on epoch 834 is: 0.564737749562691
current loss on epoch 835 is: 0.5646747183634246
current loss on epoch 836 is: 0.5646118166561209
current loss on epoch 837 is: 0.5645493022212728
current loss on epoch 838 is: 0.5644864525036066
current loss on epoch 839 is: 0.5644241197958757
current loss on epoch 840 is: 0.5643620441776208
current loss on epoch 841 is: 0.5642991520874618
current loss on epoch 842 is: 0.5642357602853615
current loss on epoch 843 is: 0.5641718288744286
current loss on epoch 844 is: 0.5641084361347496
current loss on epoch 845 is: 0.5640448096153974
current loss on epoch 846 is: 0.5639811875638381
current loss on epoch 847 is: 0.5639170718078856
current loss on epoch 848 is: 0.5638525580365975
current loss on epoch 849 is: 0.5637881073910911
current loss on epoch 850 is: 0.5637239925492235
current loss on epoch 851 is: 0.5636598852332934
current loss on epoch 852 is: 0.5635956205175897
current loss on epoch 853 is: 0.5635306478641546
current loss on epoch 854 is: 0.5634651808805033
current loss on epoch 855 is: 0.5633999442964416
current loss on epoch 856 is: 0.5633349970499145
current loss on epoch 857 is: 0.5632703203921767
current loss on epoch 858 is: 0.5632059103483844
current loss on epoch 859 is: 0.563140853003523
current loss on epoch 860 is: 0.5630763556347281
current loss on epoch 861 is: 0.5630125354084595
current loss on epoch 862 is: 0.5629491200869415
current loss on epoch 863 is: 0.5628856954199947
current loss on epoch 864 is: 0.5628222043214632
current loss on epoch 865 is: 0.5627592834784557
current loss on epoch 866 is: 0.5626970698443284
current loss on epoch 867 is: 0.5626347629771757
current loss on epoch 868 is: 0.5625726644227078
current loss on epoch 869 is: 0.5625112369120728
current loss on epoch 870 is: 0.5624498146846524
current loss on epoch 871 is: 0.562388043690667
current loss on epoch 872 is: 0.5623263490128737
current loss on epoch 873 is: 0.5622645066284123
current loss on epoch 874 is: 0.5622035787882582
current loss on epoch 875 is: 0.5621432611720455
current loss on epoch 876 is: 0.562082571950492
current loss on epoch 877 is: 0.5620216616570839
current loss on epoch 878 is: 0.5619601792003469
current loss on epoch 879 is: 0.5618988417473191
current loss on epoch 880 is: 0.5618372408846083
current loss on epoch 881 is: 0.5617751588806167
current loss on epoch 882 is: 0.5617126829737087
current loss on epoch 883 is: 0.5616506364465279
current loss on epoch 884 is: 0.5615886618371081
current loss on epoch 885 is: 0.5615272737848018
current loss on epoch 886 is: 0.5614664150603479
current loss on epoch 887 is: 0.5614062376205607
current loss on epoch 888 is: 0.5613462356929917
current loss on epoch 889 is: 0.5612862303520452
current loss on epoch 890 is: 0.5612264898803858
current loss on epoch 891 is: 0.561167394655791
current loss on epoch 892 is: 0.5611088436934932
current loss on epoch 893 is: 0.5610501628872845
current loss on epoch 894 is: 0.5609915778614405
current loss on epoch 895 is: 0.5609335488285104
current loss on epoch 896 is: 0.5608757447526918
current loss on epoch 897 is: 0.5608179008213195
current loss on epoch 898 is: 0.5607598735040024
current loss on epoch 899 is: 0.5607021401167267
current loss on epoch 900 is: 0.5606443862176687
current loss on epoch 901 is: 0.5605870986076694
current loss on epoch 902 is: 0.5605310010230576
current loss on epoch 903 is: 0.5604749778514112
current loss on epoch 904 is: 0.5604195922611127
current loss on epoch 905 is: 0.5603647260575793
current loss on epoch 906 is: 0.5603096185876655
current loss on epoch 907 is: 0.5602540134726026
current loss on epoch 908 is: 0.5601982004301869
current loss on epoch 909 is: 0.5601424058529605
current loss on epoch 910 is: 0.5600869389253227
current loss on epoch 911 is: 0.5600317006295328
current loss on epoch 912 is: 0.5599767523065668
current loss on epoch 913 is: 0.5599212740982178
current loss on epoch 914 is: 0.5598654698402007
current loss on epoch 915 is: 0.5598097811607494
current loss on epoch 916 is: 0.5597543817610592
current loss on epoch 917 is: 0.5596995089252979
current loss on epoch 918 is: 0.5596449122733502
current loss on epoch 919 is: 0.559590483446005
current loss on epoch 920 is: 0.55953624647641
current loss on epoch 921 is: 0.559481904851898
current loss on epoch 922 is: 0.5594277895554276
current loss on epoch 923 is: 0.5593735162078555
current loss on epoch 924 is: 0.5593196067080295
current loss on epoch 925 is: 0.5592659272378869
current loss on epoch 926 is: 0.5592120923885481
current loss on epoch 927 is: 0.5591582622036366
current loss on epoch 928 is: 0.5591051126099041
current loss on epoch 929 is: 0.5590525524798382
current loss on epoch 930 is: 0.559000319564587
current loss on epoch 931 is: 0.5589484759682126
current loss on epoch 932 is: 0.5588970465800734
current loss on epoch 933 is: 0.5588460283660585
current loss on epoch 934 is: 0.5587948332038721
current loss on epoch 935 is: 0.5587439311105223
current loss on epoch 936 is: 0.5586930054718694
current loss on epoch 937 is: 0.5586420443398661
current loss on epoch 938 is: 0.5585914437511451
current loss on epoch 939 is: 0.5585414758451543
current loss on epoch 940 is: 0.5584918784502179
current loss on epoch 941 is: 0.558442380547744
current loss on epoch 942 is: 0.5583927740098628
current loss on epoch 943 is: 0.5583423296149772
current loss on epoch 944 is: 0.5582917577724323
current loss on epoch 945 is: 0.5582411612062621
current loss on epoch 946 is: 0.5581904831825999
current loss on epoch 947 is: 0.5581400899767368
current loss on epoch 948 is: 0.5580897201293535
current loss on epoch 949 is: 0.5580400937700648
current loss on epoch 950 is: 0.5579907463015609
current loss on epoch 951 is: 0.5579416929901633
current loss on epoch 952 is: 0.5578927447581802
current loss on epoch 953 is: 0.5578439790415187
current loss on epoch 954 is: 0.5577954490707457
current loss on epoch 955 is: 0.5577473842559608
current loss on epoch 956 is: 0.5576997011535805
current loss on epoch 957 is: 0.5576516362891007
current loss on epoch 958 is: 0.557603259656964
current loss on epoch 959 is: 0.5575550044324875
current loss on epoch 960 is: 0.5575069947268361
current loss on epoch 961 is: 0.5574597577175986
current loss on epoch 962 is: 0.5574131645419719
current loss on epoch 963 is: 0.55736675887181
current loss on epoch 964 is: 0.5573205886150642
current loss on epoch 965 is: 0.5572745329402697
current loss on epoch 966 is: 0.5572286584978079
current loss on epoch 967 is: 0.5571823228934636
current loss on epoch 968 is: 0.5571356665445528
current loss on epoch 969 is: 0.557088584026379
current loss on epoch 970 is: 0.5570416416421131
current loss on epoch 971 is: 0.556995252468027
current loss on epoch 972 is: 0.5569488663233922
current loss on epoch 973 is: 0.5569024549890154
current loss on epoch 974 is: 0.5568566981863722
current loss on epoch 975 is: 0.5568105592862378
current loss on epoch 976 is: 0.5567646532705313
current loss on epoch 977 is: 0.5567193443568639
current loss on epoch 978 is: 0.5566741265621844
current loss on epoch 979 is: 0.5566289473613016
current loss on epoch 980 is: 0.5565831474801247
current loss on epoch 981 is: 0.5565378635589455
current loss on epoch 982 is: 0.5564926836259795
current loss on epoch 983 is: 0.5564476005192289
current loss on epoch 984 is: 0.5564025017361858
current loss on epoch 985 is: 0.5563575759391258
current loss on epoch 986 is: 0.5563128251889834
current loss on epoch 987 is: 0.5562690305955761
current loss on epoch 988 is: 0.5562260137167016
current loss on epoch 989 is: 0.5561826992837955
current loss on epoch 990 is: 0.556139694184726
current loss on epoch 991 is: 0.5560967782326076
current loss on epoch 992 is: 0.5560543797957215
current loss on epoch 993 is: 0.5560119945660328
current loss on epoch 994 is: 0.5559696694709026
current loss on epoch 995 is: 0.5559268111216963
current loss on epoch 996 is: 0.5558840519910672
current loss on epoch 997 is: 0.5558405329836023
current loss on epoch 998 is: 0.5557963522508499
current loss on epoch 999 is: 0.5557517653038585
current loss on epoch 1000 is: 0.5557079425092397
######################################
## Area Under the Curve calculation ##
######################################
Y_hat = NN.Y_hat
print(roc_auc_score(Y, Y_hat))
0.7937240335755801