www.xbdev.net
xbdev - software development
Friday April 25, 2025
Home | Contact | Support | Tutorials - Tips - Secrets - for C and C++.. Power of Machine Learning and Neural Networks ...


smallnnde: Neural Network that uses Differential Evolutionary (DE) algorithm for training

by Benjamin Kenwright


smallnnde


smallnnde is a flexible neural network implementation with DE training algorithm. Designed to be small, free, and open source. The test case example
uses a 1-3-1 network configuration to emulate a sine wave function.

• Neural network with DE training
• Support multiple layers - i.e., multilayer perceptron (MLP)
• Customizable (easy to configure for different topologies/layers/connections)
• Educational version (sometimes using libraries and packages masks the beauty of just how simple a neural network is at its heart)
• Not dependent on any libraries (vanilla C++)
• Importantly, it's a fun piece of code to play around with and learn about neural networks

The full implementation details are given below:

#include <algorithm> //std::max(..) std::min(..)
#include <iostream> // cout
#include <vector>
#include <map>
#include <math.h>
#include <time.h>


using namespace std;
#define TrainSetSize 20
#define PI 3.141592653589793238463
#define epoch 1000000

#define POPULATIONSIZE 50 

#define DBG_ASSERT(f) { if(!(f)){ __debugbreak(); }; }
double sigmoidF(double x) { return (1.0f / (1.0f std::exp(-x))); }
//double random(double min, double max) { return min + rand() / (RAND_MAX / (max - min + 1.0) + 1.0); }
double random(double mindouble max) { return min + (max min) * ((double)rand() / (double)RAND_MAX); }

double clamp(double ndouble lowerdouble upper) { return std::max(lowerstd::min(nupper)); }

struct neuron {
    
neuron() { id rand();  bias random(-11); outputF = -1; }
    
int                        id;

    
std::map<intneuron*>    incomingtargets;

    
double                      bias;
    
std::map<intdouble >    incomingweights;
    
double outputF// f(x)  forward

    
void connect(neuronndouble weight random(-11)) {
        
n->incomingtargets[this->id] = this;
        
n->incomingweights[this->id] = weight;
    }
    
double activate(const doubleinput) {
        if (
input != NULL) {
            
//this->outputB = 1;      // f'(x)
            
this->outputF = *input// f(x)
        
}
        else {
            
double sum 0// sum (x * w) + b
            
map<intneuron*>::iterator it;
            for (
it this->incomingtargets.begin(); it != this->incomingtargets.end(); it++)
            {
                
sum += (this->incomingtargets[it->first]->outputF this->incomingweights[it->first]);
            }
            
sum += this->bias;
            
this->outputF sigmoidF(sum);// f(x)
        
}
        return 
this->outputF;
    }
};

struct network {
    
vector<neuron*> inputs;//  Input Layer w/ 1 neurons
    
vector<neuron*> hiddens;// Hiddent Layer w/ 3 neurons
    
vector<neuron*> outputs;// Output Layer w/ 1 neuron
                            // Connect Input Layer to Hidden Layer
    
double error;

    
network()
    {
        
error 0;
        
inputs.push_back(new neuron()); // Input Layer w/ 1 neurons
        
hiddens.push_back(new neuron());  hiddens.push_back(new neuron()); hiddens.push_back(new neuron()); // Hiddent Layer w/ 3 neurons
        
outputs.push_back(new neuron());  // Output Layer w/ 1 neuron

        
inputs[0]->connect(hiddens[0]);        inputs[0]->connect(hiddens[1]);        inputs[0]->connect(hiddens[2]);
        
// Connect Hidden Layer to Output Layer
        
hiddens[0]->connect(outputs[0]);    hiddens[1]->connect(outputs[0]);    hiddens[2]->connect(outputs[0]);
    }
    ~
network()
    {
        
delete inputs[0];
        
delete hiddens[0];
        
delete hiddens[1];
        
delete hiddens[2];
        
delete outputs[0];
    }

    
double activate(const doubleinput, const doubleoutput) {
        for (
int i 01i++) inputs[i]->activate(input);
        for (
int i 03i++) hiddens[i]->activate(NULL);
        for (
int i 01i++) outputs[i]->activate(NULL);
        if (
output != NULLerror += abs(*output outputs[0]->outputF);
        return 
outputs[0]->outputF;
    }

    
double GetFitness() const { return error; }

    
int getDimension() const
    {
        return 
6;

    } 
// 5 neurons (5xbias), 6 connections (6xweights)

    
doublegetData(int dimensionIndex)
    {
        
// bias and incomingweights
        
neuronallneurons[] = { inputs[0], hiddens[0], hiddens[1], hiddens[2], outputs[0] };
        
int count 0;
        for (
int i 0i<5i++)
        {
            if (
count == dimensionIndex)
            {
                return 
allneurons[i]->bias;
            }
            
count++;
            for (
int k 0k<(int)allneurons[i]->incomingweights.size(); k++)
            {
                if (
count == dimensionIndex)
                {
                    return 
allneurons[i]->incomingweights[k];
                }
                
count++;
            }
        }
        
DBG_ASSERT(false);
        return 
allneurons[0]->bias// never get here!
    
}

    
void setData(int dimensionIndexdouble val)
    {
        
// bias and incomingweights
        
neuronallneurons[] = { inputs[0], hiddens[0], hiddens[1], hiddens[2], outputs[0] };
        
int count 0;
        for (
int i 0i<5i++)
        {
            if (
count == dimensionIndex)
            {
                
allneurons[i]->bias val;
                return;
            }
            
count++;
            for (
int k 0k<(int)allneurons[i]->incomingweights.size(); k++)
            {
                if (
count == dimensionIndex)
                {
                    
allneurons[i]->incomingweights[k] = val;
                    return;
                }
                
count++;
            }
        }
        
DBG_ASSERT(false);
        
//return allneurons[0]->bias; // never get here!
    
}

};


void main() {
    
srand((unsigned int)time(NULL));

    
double F  0.1;  //differential weight [0,2]
    
double CR 0.5f//crossover probability [0,1]

    
for (int test 0test 10test++)
    {
        
F  random(0.0011.9); //differential weight [0,2]
        
CR random(0.0010.9); //crossover probability [0,1]

        
vector<network*> population;// = new network*[POPULATIONSIZE];// = new network[ POPULATIONSIZE ];
        
for (int i 0POPULATIONSIZEi++)
        {
            
population.push_back(new network());
        }


        
vector<pair<doubledouble>> trainSet;    trainSet.resize(TrainSetSize);
        for (
int i 0TrainSetSizei++) { trainSet[i] = make_pair(/ (double)TrainSetSizesin((/ (double)TrainSetSize) * (2.0 PI))*0.5 0.5); }


        
//dimensionality of problem, means how many variables problem has. this case 3 (data1,data2,data3)
        
const int N population[0]->getDimension(); // 5 neurons and each neuron has connected weights and bias (should be 11 for this case)


        
for (int j 0POPULATIONSIZEj++)
        {
            for (
int ts 0ts TrainSetSizets++)
            {
                const 
double input trainSet[ts].first;
                const 
double output trainSet[ts].second;
                
population[j]->activate(&input, &output);
            }
        }

        for (
int e 0epoche++)
        {

            
//F = random(0.001, 0.5);
            //CR = random(0.01, 0.6);

            //main loop of evolution.
            
for (int j 0POPULATIONSIZEj++)
            {


                
//calculate new candidate solution

                //pick random point from population
                
int x = ::floor(random(0POPULATIONSIZE 1));

                
int abc;    //pick three different random points from population
                
do { = ::floor(random(0POPULATIONSIZE 1)); } while (== x);
                do { 
= ::floor(random(0POPULATIONSIZE 1)); } while (== || == a);
                do { 
= ::floor(random(0POPULATIONSIZE 1)); } while (== || == || == b);

                
// Pick a random index [0-Dimensionality]
                
int R = ::floor(random(0N));
                
//Compute the agent's new position

                
networkoriginal population[x];
                
networkcandidate = new network();// = original; 

                
networkindividual1 population[a];
                
networkindividual2 population[b];
                
networkindividual3 population[c];

                
//if(i==R | i<CR)
                //candidate=a+f*(b-c)
                //else
                //candidate=x

                
for (int s 0N; ++s)
                {
                    
//if (s == R || random(0, 1)<CR)
                    
if (== || random(01) < CR)
                    {
                        
//candidate->getData(s) = individual1->getData(s) + F*(individual2->getData(s) - individual3->getData(s));
                        
double val individual1->getData(s) + F*(individual2->getData(s) - individual3->getData(s));
                        
//val = val * 0.5;
                        
candidate->setData(sval);
                        
//candidate->getData(s) *= 0.5;

                        //double val = candidate->getData(s);
                        //DBG_ASSERT(val > -1000 && val < 1000);
                    
}
                    else
                    {
                        
//candidate->getData(s) = original->getData(s);
                        
double val original->getData(s);
                        
candidate->setData(sval);
                    }
                }
// End for b

                 // calculate fitness for our new candidate
                
for (int ts 0ts TrainSetSizets++)
                {
                    const 
double input trainSet[ts].first;
                    const 
double output trainSet[ts].second;
                    
candidate->activate(&input, &output);
                }

                
double candidateFitness candidate->GetFitness();
                
double originalFitness original->GetFitness();

                
// if better replace
                
if (candidateFitness originalFitness)
                {
                    
population[x] = candidate;
                    
delete original;
                }
                else
                {
                    
delete candidate;
                }

            }
// end population

#if 1 // print out the fitness as it evolves
            
{
                
int    bestFitnessIndex 0;
                
double bestFitnessValue population[bestFitnessIndex]->GetFitness();

                for (
int i 1POPULATIONSIZE; ++i)
                {
                    if (
population[i]->GetFitness() < bestFitnessValue)
                    {
                        
bestFitnessValue population[i]->GetFitness();
                        
bestFitnessIndex i;
                    }
                }
// End for i
                
cout << "epoch:" << << " fitness:" << bestFitnessValue << "\r";
            }
#endif

        
}// epoch



        
int    bestFitnessIndex 0;
        
double bestFitnessValue population[bestFitnessIndex]->GetFitness();

        for (
int i 1POPULATIONSIZE; ++i)
        {
            if (
population[i]->GetFitness() < bestFitnessValue)
            {
                
bestFitnessValue population[i]->GetFitness();
                
bestFitnessIndex i;
            }
        }
// End for i
        
DBG_ASSERT(bestFitnessIndex > -1);



#if 0
        
cout << "input, idea, predicted (y =sin(x))" << "\n";
        for (
int i 0100i++) { //Print out test results (ideal vs predicted for sin wave)
            
double input 100.0;
            
double predicted population[bestFitnessIndex].activate(&inputNULL);
            
std::cout << (input) << ", " << sin(input * (2.0 PI))*0.5 0.5 << ", " << predicted << "\n";
    } 
system("pause"); // wait for user to press a key before closing
#endif

#if 1
    
char filename[256] = { };
    
sprintf(filename"data_F=%.5f_CR=%.5f.txt"FCR);
    
FILEfp fopen(filename"wt");
    
fprintf(fp"epoch %d, populationsize: %d\n"epochPOPULATIONSIZE);
    
fprintf(fp"error %f\n"bestFitnessValue);
    
fprintf(fp"input, idea, predicted (y =sin(x))\n");
    for (
int i 0100i++) { //Print out test results (ideal vs predicted for sin wave)
        
double input 100.0;
        
double predicted population[bestFitnessIndex]->activate(&inputNULL);
        
double exact sin(input * (2.0 PI))*0.5 0.5;
        
fprintf(fp"%f %f %f \n"inputexactpredicted);
    }
    
fflush(fp);
    
fclose(fp);
    
fp NULL;

#endif

    
for (int i 0POPULATIONSIZEi++)
    {
        
delete population[i];
    }
    
population.resize(0);

    }
// end for test

    
system("pause"); // wait for user to press a key before closing 
}



Key Features


• Optionally saves the trained data to a file (compare the ideal against predicted)
• Runs through various DE parameters (CR/F) to see which provides better convergence on a result (most optimal)
• Prints out the epoch and the fitness (watch the algorithm improve as it searches for the neural network weights and biases)
• Compare the accuracy/time/computational cost with other training version (e.g., back propagation) - however, for problems that have very non-linear constraints or are based on fitness critera (no-data), these alternative training algorithms may be limited.


References


1. Deep Learning with Javascript: Example-Based Approach (Kenwright) ISBN: 979-8660010767
2. Game C++ Programming: A Practical Introduction (Kenwright) ISBN: 979-8653531095
3. smallnn: Neural Network in 99 lines of C++ (Kenwright)
4. Inverse kinematic solutions for articulated characters using massively parallel architectures and differential evolutionary algorithms (Kenwright). VRIPHYS '17: Proceedings of the 13th Workshop on Virtual Reality Interactions and Physical Simulations, April 2017
5. Neural network in combination with a differential evolutionary training algorithm for addressing ambiguous articulated inverse kinematic problems (Kenwright). SA '18: SIGGRAPH Asia 2018 Technical BriefsDecember 2018
6. smallnnde homepage



 
Advert (Support Website)

 
 Visitor:
Copyright (c) 2002-2025 xbdev.net - All rights reserved.
Designated articles, tutorials and software are the property of their respective owners.