Modified cost function.

Author: Kalvar

KRMLP.podspec

@@ -1,6 +1,6 @@
 Pod::Spec.new do |s|
  s.name = "KRMLP"
- s.version = "2.2.2"
+ s.version = "2.2.3"
  s.summary = "Deep Learning for multi-layer perceptrons neural network (MLP)."
  s.description = <<-DESC
  Machine Learning (マシンラーニング) in this project, it implemented multi-layer perceptrons neural network (ニューラルネットワーク) and Back Propagation Neural Network (BPN). It designed unlimited hidden layers to do the training tasks. This network can be used in products recommendation (おすすめの商品), user behavior analysis (ユーザーの行動分析), data mining (データマイニング) and data analysis (データ分析).

ML/KRMLP.m

@@ -43,9 +43,8 @@ - (void)setupNetworkActivation
 
 - (void)setupCostFunction
 {
- KRMLPCost *cost = self.cost;
- cost.patternsCount = self.patternsCount;
- cost.outputsCount = self.outputNetsCount;
+ KRMLPCost *cost = self.cost;
+ [cost removeRecords];
 }
 
 - (void)initializeSettings
@@ -95,8 +94,8 @@ - (void)trainingWithInputs:(NSArray <NSNumber *> *)inputs targets:(NSArray <NSNu
 
  // 多分類的 output layer to hidden layer 跟 hidden layer to hidden layer / input layer 的 delta value (誤差值) 算法都一樣。
  // Recording cost value, btw, the delta-values of output-layer-nets are recorded in the output-nets self (net.deltaValue).
- KRMLPCost *cost = self.cost;
- cost.costValue = [outputLayer calculateCostAndDeltaWithTargets:targets];
+ [outputLayer calculateDeltasWithTargets:targets];
+ [self.cost addOutputs:outputLayer.outputs targets:targets];
 
  // Reversing and looping hiddenLayers to start in calculating the delta values from output-layer to hidden layers (that includes hidden layer to hidden layer).
  // lastDeltas the initial value is delta-values of output-nets, the delta-value comes from net.deltaValue.
@@ -132,6 +131,7 @@ - (void)trainingWithInputs:(NSArray <NSNumber *> *)inputs targets:(NSArray <NSNu
 
 - (void)iterationTraining
 {
+ [self.cost removeRecords];
  self.currentIteration += 1;
 
  // The self.patterns is getter of inputLayer.nets.

ML/Layers/KRMLPOutputLayer/KRMLPCost.h

@@ -10,12 +10,15 @@
 
 @interface KRMLPCost : NSObject
 
-@property (nonatomic, assign) double costValue; // The (targetValue - outputValue)^2.
-@property (nonatomic, assign) NSInteger patternsCount; // Patterns counting number.
-@property (nonatomic, assign) NSInteger outputsCount; // Output nets counting number.
-@property (nonatomic, readonly) double mse; // The costValue / (patternsCount * outputsCount) / 2.0f
-@property (nonatomic, readonly) double rmse;  // The costValue / (patternsCount * outputsCount)
+@property (nonatomic, readonly) NSInteger patternsCount; // Patterns counting number.
+@property (nonatomic, readonly) NSInteger outputsCount; // Output nets counting number.
+@property (nonatomic, readonly) double mse;  // The costValue / (patternsCount * outputsCount) / 2.0f
+@property (nonatomic, readonly) double rmse;  // The costValue / (patternsCount * outputsCount)
+@property (nonatomic, readonly) double crossEntropy; // Not all situations can use Cross-Entropy to optimize, in some cases the MSE better than it.
 
 - (instancetype)init;
 
+- (void)addOutputs:(NSArray <NSNumber *> *)outputs targets:(NSArray <NSNumber *> *)targets;
+- (void)removeRecords;
+
 @end

ML/Layers/KRMLPOutputLayer/KRMLPCost.m

@@ -8,11 +8,19 @@
 
 #import "KRMLPCost.h"
 
+@interface KRMLPCost ()
+
+@property (nonatomic, readonly) double costValue; // The (targetValue - outputValue)^2.
+@property (nonatomic) NSMutableArray <NSArray <NSNumber *> *> *outputs; // The outputs of patterns in this iteration.
+@property (nonatomic) NSMutableArray <NSArray <NSNumber *> *> *targets; // The targets of patterns in this iteration.
+
+@end
+
 @implementation KRMLPCost (Checks)
 
 - (BOOL)canCalculate
 {
- return ( self.patternsCount != 0.0f && self.outputsCount != 0.0f );
+ return (self.patternsCount != 0 && self.outputsCount != 0);
 }
 
 @end
@@ -24,25 +32,82 @@ - (instancetype)init
  self = [super init];
  if( self )
  {
- _costValue = 0.0f;
- _patternsCount = 0.0f;
- _outputsCount = 0.0f;
+ _outputs = [[NSMutableArray alloc] init];
+ _targets = [[NSMutableArray alloc] init];
  }
  return self;
 }
 
+- (void)addOutputs:(NSArray<NSNumber *> *)outputs targets:(NSArray<NSNumber *> *)targets
+{
+ [_outputs addObject:[outputs copy]];
+ [_targets addObject:[targets copy]];
+}
+
+- (void)removeRecords
+{
+ [_outputs removeAllObjects];
+ [_targets removeAllObjects];
+}
+
 #pragma mark - Getters
+- (double)costValue
+{
+ __block NSArray <NSArray <NSNumber *> *> *targets = _targets;
+ __block double costValue = 0.0f;
+ [_outputs enumerateObjectsUsingBlock:^(NSArray<NSNumber *> * _Nonnull patternOutputs, NSUInteger idx, BOOL * _Nonnull stop) {
+ __block NSArray <NSNumber *> *patternTargets = [targets objectAtIndex:idx];
+ [patternOutputs enumerateObjectsUsingBlock:^(NSNumber * _Nonnull netOutput, NSUInteger idx, BOOL * _Nonnull stop) {
+ double outputValue = [netOutput doubleValue];
+ double targetValue = [[patternTargets objectAtIndex:idx] doubleValue];
+ double outputError = targetValue - outputValue;
+ // 計算 MSE, RMSE 共同使用的總和誤差值 : SUM(error^2)
+ costValue += (outputError * outputError);
+ }];
+ }];
+ return costValue;
+}
+
+- (NSInteger)patternsCount
+{
+ return [_outputs count];
+}
+
+- (NSInteger)outputsCount
+{
+ return [[_outputs firstObject] count];
+}
+
 // e.g. Cost Value = SUM( ( d(j) - y(j) ) ^ 2 ) / ( [patterns count] * [outputNets count] ) * 0.5f
 // MSE (均方誤差) = Cost Value / (訓練範例數目 x 數出層神經元數目) / 2
-// RMSE (均方根誤差) = sqrt(Cost Value / (訓練範例數目 x 數出層神經元數目) / 2)
+// RMSE (均方根誤差) = sqrt(Cost Value / (訓練範例數目 x 數出層神經元數目))
 - (double)mse
 {
- return [self canCalculate] ? _costValue / (_patternsCount * _outputsCount) * 0.5f : NSNotFound;
+ return [self canCalculate] ? self.costValue / (self.patternsCount * self.outputsCount) * 0.5f : NSNotFound;
 }
 
 - (double)rmse
 {
- return [self canCalculate] ? sqrt(_costValue / (_patternsCount * _outputsCount) * 0.5f) : NSNotFound;
+ return [self canCalculate] ? sqrt(self.costValue / (self.patternsCount * self.outputsCount)) : NSNotFound;
+}
+
+- (double)crossEntropy
+{
+ __block NSArray <NSArray <NSNumber *> *> *targets = _targets;
+ __block double iterationEntropy = 0.0f;
+ __block NSInteger outputCount = self.outputsCount;
+ [_outputs enumerateObjectsUsingBlock:^(NSArray<NSNumber *> * _Nonnull patternOutputs, NSUInteger idx, BOOL * _Nonnull stop) {
+ __block NSArray <NSNumber *> *patternTargets = [targets objectAtIndex:idx];
+ __block double patternEntropy = 0.0f;
+ [patternOutputs enumerateObjectsUsingBlock:^(NSNumber * _Nonnull netOutput, NSUInteger idx, BOOL * _Nonnull stop) {
+ double outputValue = [netOutput doubleValue];
+ double targetValue = [[patternTargets objectAtIndex:idx] doubleValue];
+ double entropy = (targetValue * log(outputValue)) + ((1.0f - targetValue) * log(1.0f - outputValue));
+ patternEntropy += entropy;
+ }];
+ iterationEntropy += -(patternEntropy / outputCount);
+ }];
+ return (iterationEntropy / self.patternsCount);
 }
 
 @end

ML/Layers/KRMLPOutputLayer/KRMLPOutputLayer.h

@@ -19,7 +19,7 @@
 - (void)addNetsFromArray:(NSArray <KRMLPOutputNet *> *)layerNets;
 - (NSArray <NSNumber *> *)directOutputsWithInputs:(NSArray <NSNumber *> *)inputs;
 - (NSArray <NSNumber *> *)layerOutputsWithInputs:(NSArray <NSNumber *> *)inputs;
-- (double)calculateCostAndDeltaWithTargets:(NSArray <NSNumber *> *)targets;
+- (void)calculateDeltasWithTargets:(NSArray <NSNumber *> *)targets;
 - (void)updateWithHiddenLayerOutputs:(NSArray <NSNumber *> *)layerOutputs learningRate:(double)learningRate;
 
 @end

ML/Layers/KRMLPOutputLayer/KRMLPOutputLayer.m

@@ -71,23 +71,19 @@ - (void)addNetsFromArray:(NSArray<KRMLPOutputNet *> *)layerNets
 
 // 計算修正 Weights 與 Biases 用的 Delta Values 和當前 Pattern 的 Cost Value (網路輸出誤差值) for Cost Function.
 // Returns the cost function value of this pattern.
-- (double)calculateCostAndDeltaWithTargets:(NSArray <NSNumber *> *)targets
+- (void)calculateDeltasWithTargets:(NSArray <NSNumber *> *)targets
 {
  __weak typeof(self) weakSelf = self;
- __block double costValue = 0.0f;
  [_nets enumerateObjectsUsingBlock:^(KRMLPOutputNet * _Nonnull outputNet, NSUInteger idx, BOOL * _Nonnull stop) {
  __strong typeof(weakSelf) strongSelf = weakSelf;
  double targetValue = [[targets objectAtIndex:idx] doubleValue];
  double outputValue = [[strongSelf.outputs objectAtIndex:idx] doubleValue];
  double outputError = targetValue - outputValue; // d(j) - y(j) = d(j) - f(net(j))
- // 計算 MSE, RMSE 共同使用的總和誤差值 : SUM(error^2)
- costValue += ( outputError * outputError );
  // a is Partial Derivative simple.
  // Delta Value 是由 a(E)/a(wij) = ( d(j) - f(net(j)) ) * -f'(net(j)) = ( d(j) - y(j) ) * -f'( y(j) )
  // Output Net(j) 的 Delta 輸出誤差由於後續修正權重公式的關係, 會將負號消去, 故這裡先直接將負號消去: ( d(j) - y(j) ) * f'( y(j) )
  outputNet.deltaValue = ( outputError * outputNet.outputPartialDerivative );
  }];
- return costValue;
 }
 
 // Updating the weights and biases with last hidden layer outputs.

README.md

@@ -7,7 +7,7 @@ Machine Learning (マシンラーニング) in this project, it implemented mult
 
 ```ruby
 platform :ios, '9.0'
-pod "KRMLP", "~> 2.2.2"
+pod "KRMLP", "~> 2.2.3"
 ```
 
 ## How to use
@@ -274,7 +274,7 @@ QuickProp:
 
 ## Version
 
-V2.2.2
+V2.2.3
 
 ## License