1+ import numpy as np
2+ import math
3+ from statistics import mean
4+
5+ class NeuralNetwork ():
6+
7+ def __init__ (self , X = None , y = None , hiddenLayers = 2 , neuronsEachLayer = 2 , learning_rate = 0.01 ,
8+ epochs = 5 , method = 'Linear' , tol = 0.1 , batch_size = 32 ):
9+ self .weights = None
10+ self .activationHidden = self .sigmoid
11+ if method == 'Linear' :
12+ self .activationOut = self .linear
13+ self .derivate_out = self .linear_der
14+ elif method == 'Logistic' :
15+ self .activationOut = self .sigmoid
16+ self .derivate_out = self .sigmoid_der
17+ self .derivate_rest = self .sigmoid_der
18+ self .X = X
19+ self .Y = y
20+ self .hiddenLayers = hiddenLayers
21+ self .neuronsEachLayer = neuronsEachLayer
22+ self .learning_rate = learning_rate
23+ self .epochs = epochs
24+ self .tol = tol
25+ self .batch_size = batch_size
26+
27+ def weightsInitialisation (self ):
28+ #Initialising a numpy array of dim(hiddenlayers, neurons) to store weights
29+ self .weights = np .empty ((self .hiddenLayers , self .neuronsEachLayer ), dtype = object )
30+
31+ for i in range (self .hiddenLayers ):
32+ for j in range (self .neuronsEachLayer ):
33+ #first hidden layer
34+ if i == 0 :
35+ self .weights [i ,j ] = np .random .normal (0 ,0.4 , size = 1 + self .X .shape [1 ])
36+ #rest hidden layers
37+ else :
38+ self .weights [i ,j ] = np .random .normal (0 ,0.4 , size = 1 + self .neuronsEachLayer )
39+ #Weights for the final output layer
40+ self .outputLayerWeights = np .random .normal (0 ,0.4 , size = 1 + self .neuronsEachLayer )
41+
42+ def gradientInitialisation (self ):
43+ self .gradient = np .empty ((self .hiddenLayers , self .neuronsEachLayer ), dtype = object )
44+ for i in range (self .hiddenLayers ):
45+ for j in range (self .neuronsEachLayer ):
46+ #first hidden layer
47+ if i == 0 :
48+ self .gradient [i ,j ] = np .zeros (1 + self .X .shape [1 ])
49+ #rest hidden layers
50+ else :
51+ self .gradient [i ,j ] = np .zeros (1 + self .neuronsEachLayer )
52+ self .gradientOutputLayer = [0 ] * len (self .outputLayerWeights )
53+
54+ def sigmoid (self ,x ):
55+ if x < 0 :
56+ return 1 - 1 / (1 + math .exp (x ))
57+ else :
58+ return 1 / (1 + math .exp (- x ))
59+
60+ def linear (self ,x ):
61+ return x
62+
63+ def sigmoid_der (self ,x ):
64+ return self .sigmoid (x ) * (1 - self .sigmoid (x ))
65+
66+ def linear_der (self , x ):
67+ return 1.0
68+
69+ def squareErrorLoss (self ,x ,y ):
70+ return (self .feedForward (x ) - y )** 2
71+
72+ def error (self , X , y ):
73+ pred = []
74+ for i in X :
75+ pred .append (self .feedForward (i ))
76+ return mean ([(a_i - b_i )** 2 for a_i , b_i in zip (pred , y )])
77+
78+ def predict (self ,X ):
79+ pred = []
80+ for i in X :
81+ pred .append (self .feedForward (i ))
82+ return pred
83+
84+ def feedForward (self , x ):
85+ self .x = np .append (x , 1.0 )
86+ self .out = np .empty (shape = (self .hiddenLayers , self .neuronsEachLayer ))
87+ for i in range (self .hiddenLayers + 1 ):
88+ outputFromCurrLayer = []
89+ #For first Layer
90+ if i == 0 :
91+ for j in range (self .neuronsEachLayer ):
92+ z = self .activationHidden (np .dot (self .weights [i ,j ],self .x ))
93+ self .out [i ,j ] = z
94+ outputFromCurrLayer .append (z )
95+ outputFromCurrLayer .append (1.0 )
96+ outputFromPrevLayer = outputFromCurrLayer .copy ()
97+ #Output Layer
98+ elif i == self .hiddenLayers :
99+ return self .activationOut (np .dot (self .outputLayerWeights ,outputFromPrevLayer ))
100+ #Rest all Layers
101+ else :
102+ for j in range (self .neuronsEachLayer ):
103+ z = self .activationHidden (np .dot (self .weights [i ,j ],outputFromPrevLayer ))
104+ self .out [i ,j ] = z
105+ outputFromCurrLayer .append (z )
106+ outputFromCurrLayer .append (1.0 )
107+ outputFromPrevLayer = outputFromCurrLayer .copy ()
108+
109+ def backProp (self , pred , n , actual ):
110+ #Weight updation for Output Layer
111+ delta = []
112+ der_outter_layer = self .derivate_out (np .dot (np .append (self .out [self .hiddenLayers - 1 ], 1.0 ) , self .outputLayerWeights ))
113+ for i in range (len (self .outputLayerWeights )):
114+ if i == len (self .outputLayerWeights ) - 1 :
115+ self .gradientOutputLayer [i ] = self .gradientOutputLayer [i ] + ((2.0 / n ) * (pred - actual ) * der_outter_layer * 1 )
116+ else :
117+ d = (2.0 / n ) * (pred - actual ) * der_outter_layer * self .outputLayerWeights [i ]
118+ self .gradientOutputLayer [i ] = self .gradientOutputLayer [i ] + ((2.0 / n ) * (pred - actual ) * der_outter_layer * self .out [self .hiddenLayers - 1 , i ])
119+ delta .append (d )
120+ #For all other Layers
121+ for l in reversed (range (self .hiddenLayers )):
122+ delta_forward = delta .copy ()
123+ delta = [0 ] * self .neuronsEachLayer
124+ #For the first layer
125+ if l == 0 :
126+ for j in range (self .neuronsEachLayer ):
127+ der_layer = self .derivate_rest (np .dot (self .x , self .weights [l ,j ]))
128+ for i in range (len (self .weights [l ,j ])):
129+ if i == len (self .weights [l ,j ]) - 1 :
130+ self .gradient [l ,j ][i ] = self .gradient [l ,j ][i ] + ((1.0 / n ) * delta_forward [j ] * der_layer * 1.0 )
131+ else :
132+ self .gradient [l ,j ][i ] = self .gradient [l ,j ][i ] + ((1.0 / n ) * delta_forward [j ] * der_layer * self .x [i ])
133+ #Rest all the layers
134+ else :
135+ for j in range (self .neuronsEachLayer ):
136+ der_layer = self .derivate_rest (np .dot (np .append (self .out [l - 1 ], 1.0 ) , self .weights [l ,j ]))
137+ for i in range (len (self .weights [l ,j ])):
138+ if i == len (self .weights [l ,j ]) - 1 :
139+ self .gradient [l ,j ][i ] = self .gradient [l ,j ][i ] + ((1.0 / n ) * delta_forward [j ] * der_layer * 1.0 )
140+ else :
141+ d = (1.0 / n ) * delta_forward [j ] * der_layer * self .weights [l ,j ][i ]
142+ delta [i ] = delta [i ] + d
143+ self .gradient [l ,j ][i ] = self .gradient [l ,j ][i ] + (delta_forward [j ] * der_layer * self .out [l - 1 , i ])
144+
145+ def updateWeights (self , n ):
146+ for i in range (len (self .outputLayerWeights )):
147+ self .outputLayerWeights [i ] = self .outputLayerWeights [i ] - (self .learning_rate * (1.0 / n ) * self .gradientOutputLayer [i ])
148+ #For all other Layers
149+ for l in reversed (range (self .hiddenLayers )):
150+ for j in range (self .neuronsEachLayer ):
151+ for i in range (len (self .weights [l ,j ])):
152+ self .weights [l ,j ][i ] = self .weights [l ,j ][i ] - (self .learning_rate * (1.0 / n ) * self .gradient [l ,j ][i ])
153+
154+
155+ def fit (self ,X ,y ,X_val = None , Y_val = None ):
156+ self .X = X
157+ self .y = y
158+ self .weightsInitialisation ()
159+ self .gradientInitialisation ()
160+ i = 0
161+ error_val_old = - 1
162+ tol_count = 0
163+ while i < self .epochs :
164+ for j in range (len (X )):
165+ if j % self .batch_size == 0 and j != 0 or j == len (X ) - 1 :
166+ if j == len (X ) - 1 :
167+ self .updateWeights (j % self .batch_size )
168+ else :
169+ self .updateWeights (self .batch_size )
170+ self .gradientInitialisation ()
171+ p = self .feedForward (X [j ])
172+ self .backProp (p ,1 ,y [j ])
173+ else :
174+ p = self .feedForward (X [j ])
175+ self .backProp (p ,1 ,y [j ])
176+ if X_val is not None and Y_val is not None :
177+ error_curr_val = self .error (X_val , Y_val )
178+ print ("Epoch : {} and MSE_Train : {} and MSE_Val : {}" .format (i , self .error (X ,y ), error_curr_val ))
179+ if abs (error_val_old - error_curr_val ) < self .tol :
180+ tol_count = tol_count + 1
181+ error_val_old = error_curr_val
182+ if tol_count > 1 :
183+ print ("Stopping as validation error did not improve more than tol = {} for 2 iterations" .format (self .tol ))
184+ break
185+ else :
186+ tol_count = 0
187+ error_val_old = error_curr_val
188+ else :
189+ print ("Epoch : {} and MSE : {}" .format (i , self .error (X ,y )))
190+ i = i + 1
0 commit comments