I'm trying to implement perceptron on Objective-C. Now i'm facing the problem that after some random amount of epochs, when error's tending to its minimum or maximum, network is paralysis, and after some more epochs all weights becomes very big or very small numbers, and output and propagation of network becomes NaN. What could be the problem?
My code is here, the main method is learn: Neuron.h
@interface Neuron : NSObject <NSCoding>
@property (nonatomic) double output;
@property (strong,nonatomic) NSMutableArray *weights;
@property (nonatomic) double propagation;
@end
Web.h
#import "Neuron.h"
@interface Web : NSObject
@property (strong,nonatomic) NSMutableArray *arrayOfLayers;
-(void)setupWebWithNumberOfNeutrons:(NSInteger)number inputArray:(NSArray*)input;
-(void)addToLearningData:(NSArray *)array;
-(void)saveLearningArrayWithPath:(NSString *)path;
-(void)learn;
-(void)setupWeb;
@end
Web.m
@interface Web() <NSCoding>
@property (nonatomic) NSInteger numberOfHiddenLayerNeutrons;
@property (nonatomic) double output;
@property (nonatomic) double propagation;
@property (nonatomic) double answer;
@property (strong,nonatomic) NSMutableArray *learningArray;
@property (strong,nonatomic) NSMutableArray *learningData;
@property (strong,nonatomic) NSMutableArray *epocheLearningArrays;
@property (strong,nonatomic) NSMutableArray *epocheTestingArrays;
@end
const NSInteger kNumberOfHiddenNeurons = 20;
const NSInteger kNumberOfEpocheLearningArray = 70;
const NSInteger kMaxEpocheCount = 100;
@implementation Web
#pragma mark - Properties
-(NSMutableArray *)learningArray
{
if (!_learningArray) {
_learningArray = [[NSMutableArray alloc] init];
}
return _learningArray;
}
-(NSMutableArray *)learningData {
if (!_learningData) {
_learningData = [NSMutableArray array];
}
return _learningData;
}
-(NSMutableArray *)epocheLearningArrays {
if (!_epocheLearningArrays) {
_epocheLearningArrays = [NSMutableArray array];
}
return _epocheLearningArrays;
}
-(NSMutableArray *)epocheTestingArrays {
if (!_epocheTestingArrays) {
_epocheTestingArrays = [NSMutableArray array];
}
return _epocheTestingArrays;
}
-(NSMutableArray *)arrayOfLayers
{
if (!_arrayOfLayers) {
_arrayOfLayers = [[NSMutableArray alloc] init];
}
return _arrayOfLayers;
}
-(void)addToLearningData:(NSArray *)array
{
[self.learningData addObject:array];
[self saveLearningDataWithPath:nil];
}
#pragma mark - Setup
-(void)setupWeb {
NSMutableArray *arrayOfInputNeurons = [[NSMutableArray alloc] init];
for (int i=0; i < 100; i++) {
Neuron *neuron = [[Neuron alloc] init];
[arrayOfInputNeurons addObject:neuron];
}
[self.arrayOfLayers addObject:arrayOfInputNeurons];
NSMutableArray *arrayOfFirstHiddenLayerNeurons = [[NSMutableArray alloc] init];
for (int i=0; i < kNumberOfHiddenNeurons; i++) {
Neuron *neuron = [[Neuron alloc] init];
[arrayOfFirstHiddenLayerNeurons addObject:neuron];
}
[self.arrayOfLayers addObject:arrayOfFirstHiddenLayerNeurons];
NSMutableArray *arrayOfSeconHiddenLayerNeurons = [[NSMutableArray alloc] init];
Neuron *outputNeuron = [[Neuron alloc] init];
[arrayOfSeconHiddenLayerNeurons addObject:outputNeuron];
[self.arrayOfLayers addObject:arrayOfSeconHiddenLayerNeurons];
[self setRandomWeights];
}
-(void)setupWebWithNumberOfNeutrons:(NSInteger)number inputArray:(NSArray*)input
{
self.output = 1;
NSMutableArray *arrayOfInputNeurons = [[NSMutableArray alloc] init];
for (NSNumber *state in input) {
Neuron *neuron = [[Neuron alloc] init];
neuron.output = state.intValue;
[arrayOfInputNeurons addObject:neuron];
}
[self.arrayOfLayers addObject:arrayOfInputNeurons];
NSMutableArray *arrayOfFirstHiddenLayerNeurons = [[NSMutableArray alloc] init];
for (int i=0; i < number; i++) {
Neuron *neuron = [[Neuron alloc] init];
[arrayOfFirstHiddenLayerNeurons addObject:neuron];
}
[self.arrayOfLayers addObject:arrayOfFirstHiddenLayerNeurons];
NSMutableArray *arrayOfSeconHiddenLayerNeurons = [[NSMutableArray alloc] init];
Neuron *outputNeuron = [[Neuron alloc] init];
[arrayOfSeconHiddenLayerNeurons addObject:outputNeuron];
[self.arrayOfLayers addObject:arrayOfSeconHiddenLayerNeurons];
if (!self.propagation) {
[self setRandomWeights];
}
[self forwardPass];
Neuron *neuron = (Neuron *)self.arrayOfLayers[2][0];
self.answer = -1;
[self calculatePropagation:neuron];
[self backPass];
[self resetWeights];
}
-(void)setRandomWeights
{
for (NSInteger i=0; i<[self.arrayOfLayers count]-1; i++) {
for (Neuron *neuron in self.arrayOfLayers[i]) {
NSMutableArray *weights = [NSMutableArray array];
for (NSInteger j=0; j<[self.arrayOfLayers[i+1] count]; j++) {
[weights addObject:@((((float) rand() / RAND_MAX) * 1) -0.5)];
}
neuron.weights = weights;
}
}
}
-(void)setInputLayer {
for (int i=0; i<100; i++) {
Neuron *neuron = self.arrayOfLayers[0][i];
NSNumber *output = self.learningArray[i];
neuron.output = output.doubleValue;
}
}
#pragma mark - Algorythm
-(void)learn {
double currentAnswer;
NSMutableArray *shuffledData = [self.learningData mutableCopy];
[self shuffleArray:shuffledData];
[self setRandomWeights];
int countOfLearningArray = (int) (0.7*[self.learningData count]);
for (int k=0; k<countOfLearningArray; k++) {
[self.epocheLearningArrays addObject:[shuffledData objectAtIndex:k]];
}
for (int k=countOfLearningArray; k<[shuffledData count]; k++) {
[self.epocheTestingArrays addObject:[shuffledData objectAtIndex:k]];
}
NSNumber *learningPropagation=@(-100);
NSNumber *testPropagation=@(100);
NSInteger epocheCount=0;
while (fabs(learningPropagation.doubleValue-testPropagation.doubleValue)>0.001 && epocheCount<kMaxEpocheCount) {
learningPropagation = @0;
for (NSArray *learningArray in self.epocheLearningArrays) {
self.learningArray = [learningArray mutableCopy];
NSNumber *lastObject = [learningArray lastObject];
currentAnswer = lastObject.doubleValue;
[self.learningArray removeLastObject];
self.answer = currentAnswer;
[self setInputLayer];
[self forwardPass];
[self calculatePropagation:self.arrayOfLayers[2][0]];
if (fabs(self.output-self.answer)>0.0001) {
[self backPass];
[self resetWeights];
}
learningPropagation = @(learningPropagation.doubleValue + self.propagation);
}
learningPropagation = @(learningPropagation.doubleValue/[self.epocheLearningArrays count]);
testPropagation = @0;
for (NSArray *testArray in self.epocheTestingArrays) {
self.learningArray = [testArray mutableCopy];
NSNumber *lastObject = [testArray lastObject];
currentAnswer = lastObject.doubleValue;
[self.learningArray removeLastObject];
self.answer = currentAnswer;
[self setInputLayer];
[self forwardPass];
[self calculatePropagation:self.arrayOfLayers[2][0]];
testPropagation = @(testPropagation.doubleValue + self.propagation);
}
testPropagation = @(testPropagation.doubleValue/[self.epocheTestingArrays count]);
epocheCount++;
}
}
-(void)calculatePropagation:(Neuron *)neuron
{
self.propagation= 0.5*pow((neuron.output - self.answer), 2);
}
#define alpha 0.5
-(void)forwardPass
{
for (int i=0; i<[self.arrayOfLayers count]-1; i++) {
for (int j=0; j<[self.arrayOfLayers[i+1] count]; j++) {
double sum=0;
for (int z=0; z<[self.arrayOfLayers[i] count]; z++) {
Neuron *neuron = (Neuron *)self.arrayOfLayers[i][z];
sum+=[(NSNumber *)neuron.weights[j] doubleValue]*neuron.output;
}
Neuron *neuron = (Neuron *)self.arrayOfLayers[i+1][j];
neuron.output = tanh(alpha*sum);
}
}
Neuron *outputNeuron = self.arrayOfLayers[2][0];
self.output = outputNeuron.output;
}
-(void)backPass
{
for (NSInteger i=[self.arrayOfLayers count]-1; i>=0; i--) {
for (int j=0; j<[self.arrayOfLayers[i] count]; j++) {
double temp=0;
if (i==[self.arrayOfLayers count]-1) {
Neuron *neuron = (Neuron *)self.arrayOfLayers[i][j];
temp = neuron.output-self.answer;
neuron.propagation = temp;
} else {
for (int z=0; z<[self.arrayOfLayers[i+1] count]; z++) {
Neuron *neuron1 = (Neuron *)self.arrayOfLayers[i+1][z];
Neuron *neuron2 = (Neuron *)self.arrayOfLayers[i][j];
temp = neuron1.propagation * [(NSNumber *)neuron2.weights[z] doubleValue];
}
Neuron *neuron = (Neuron *)self.arrayOfLayers[i][j];
neuron.propagation = temp;
}
}
}
}
#define gamma 0.01
-(void)resetWeights
{
for (int i=0; i<[self.arrayOfLayers count]-1; i++) {
for (int j=0; j<[self.arrayOfLayers[i] count]; j++) {
Neuron *neuron = (Neuron *)self.arrayOfLayers[i][j];
for (int z=0; z<[self.arrayOfLayers[i+1] count]; z++) {
neuron.weights[z] = @([(NSNumber *)neuron.weights[z] doubleValue]- gamma * neuron.propagation * neuron.output);
}
}
}
}
-(void)shuffleArray:(NSMutableArray *)array {
NSUInteger count = [array count];
for (NSUInteger i = 0; i < count; ++i) {
NSInteger remainingCount = count - i;
NSInteger exchangeIndex = i + arc4random_uniform((u_int32_t )remainingCount);
[array exchangeObjectAtIndex:i withObjectAtIndex:exchangeIndex];
}
}