Core network library for a deep neural network. More...

#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdarg.h>
#include <stdbool.h>
#include "util/mnist-utils.h"
#include "util/screen.h"
#include "dnn.h"

Macros
#define	OUT_OF_RANGE -1

Functions
int	getLayerColumnCount (LayerDefinition *layerDef)
	Returns the number of columns in a layer. More...

int	getLayerNodeCount (LayerDefinition *layerDef)
	Returns the number of nodes in a layer. More...

int	getNodeBackwardConnectionCount (LayerDefinition *layerDef)
	Returns the number of backward connections of a NODE (not of a layer) More...

int	getNodeForwardConnectionCount (LayerDefinition *layerDef)
	Returns the number of forward connections of a NODE (not of a layer) More...

int	getLayerWeightCount (LayerDefinition *layerDef)
	Returns the number of weights for a layer (based on a given layer definition) More...

int	getColumnCount (LayerDefinition *layerDef)
	Returns the number of columns in a layer (based on a give layer definition) More...

ByteSize	getLayerWeightBlockSize (LayerDefinition *layerDef)
	Returns the memory (byte) size of the weights block for a specific layer. More...

ByteSize	getNetworkWeightBlockSize (int layerCount, LayerDefinition *layerDefs)
	Returns the memory size of the network's weights block based on a given array of layer definitions. More...

ByteSize	getNodeSize (LayerDefinition *layerDef)
	Returns the memory (byte) size of a node based on a given layer definition. More...

ByteSize	getColumnSize (LayerDefinition *layerDef)
	Returns the memory (byte) size of a column based on a given layer definition. More...

ByteSize	getLayerSize (LayerDefinition *layerDef)
	Returns the memory (byte) size of a specific layer based on a given layer definition. More...

ByteSize	getNetworkSize (int layerCount, LayerDefinition *layerDefs)
	Returns the memory size of the network based on an array of layer definitions and weightBlockSize. More...

Node *	getColumnNode (Column *column, int nodeId)
	Returns a pointer to a specific node defined by its id from a given layer. More...

Column *	getLayerColumn (Layer *layer, int columnId)
	Returns a pointer to a specific column defined by its id. More...

Node *	getNetworkNode (Layer *layer, int columnId, int nodeId)
	Returns a pointer to a specific node defined by its layer, column and node id. More...

Layer *	getNetworkLayer (Network *nn, int layerId)
	Returns a pointer to a specific layer defined by its id from the network. More...

Weight	getDerivative (Weight outVal, ActFctType actType)
	Returns the result of applying the given outputValue to the derivate of the activation function. More...

void	updateNodeWeights (Node *updateNode, double learningRate)
	Updates a node's weights based on given learning rate. More...

double	calcNodeError (Node *thisNode)
	Returns the total error of a node by adding up all the partial errors from the following layer. More...

void	backPropagateLayer (Network *nn, int layerId)
	Back propagates network error to hidden layer. More...

void	backPropagateOutputLayer (Network *nn, int targetClassification)
	Calculates the error (difference of desired classification vs actual node output) of each output node and back propagates the error in the output layer to the previous layer. More...

void	backPropagateNetwork (Network *nn, int targetClassification)
	Backpropagates the output nodes' errors from output layer backwards to first layer. More...

void	activateNode (Node *node, ActFctType actType)
	Performs an activiation function to a specified node. More...

void	calcNodeOutput (Node *node)
	Calculates the output value of a specified node. More...

void	calcNetworkLayer (Layer *layer)
	Calculates the output values of all nodes of a given layer. More...

void	feedForwardNetwork (Network *nn)
	Feeds forward (=calculating a node's output value and applying an activation function) layer by layer. More...

void	feedInput (Network nn, Vector v)
	Feeds some Vector data into the INPUT layer of the network. More...

int	getNetworkClassification (Network *nn)
	Returns the network's classification of the input image by choosing the node with the hightest output. More...

void	initNetworkWeights (Network *nn)

int	calcStride (int tgtWidth, int filter, int srcWidth)
	Calculates the stride (number of nodes/columns that are skipped) in a convolutional kernel. More...

void	calcFilterColumnIds (Layer srcLayer, int srcColId, Layer tgtLayer, Vector *filterColIds)
	Returns an array of FILTER-many column ids representing a moving x*y kernel window in the target layer. More...

void	initNetworkBackwardConnectionsConvNode (Node node, int srcLevel, Weight srcLayerWeightPtr, Layer targetLayer, Vector filterColIds, Weight *nullWeight)
	Initializes a single convolutional node by setting its connections weights pointers. More...

void	initNetworkBackwardConnectionsFCNode (Node thisNode, Layer prevLayer, Weight *nodeWeightPtr)
	Initializes a node of a normal, fully connected node. More...

void	initNetworkForwardConnectionsAnyNode (Node thisNode, Layer nextLayer)

void	initNetworkForwardConnections (Network *nn, int layerId)

Vector *	createFilterColumnIds (Layer thisLayer, int columnId, Layer prevLayer)
	Creates and returns an array of FILTER-many column ids representing a moving x*y kernel window. More...

void	setNetworkNodeDefaults (Layer thisLayer, Column column, Node node, Weight nullWeight)
	Sets default values for a node during its initialization. More...

void	initNetworkNodes (Network *nn, int layerId, int columnId)
	Initializes the nodes in a given network column. More...

void	initNetworkColumns (Network *nn, int layerId)
	Initializes a network layer's column/node structure and sets detault values. More...

void	initNetworkLayer (Network nn, int layerId, LayerDefinition layerDefs)
	Initializes a network layer by creating the column/node structure and sets detault values. More...

void	initNetwork (Network nn, int layerCount, LayerDefinition layerDefs)

void	setNetworkDefaults (Network nn, int layerCount, LayerDefinition layerDefs, ByteSize netSize)
	Sets the network's default values for size, layerCount,. More...

Network *	createNetwork (int layerCount, LayerDefinition *layerDefs)
	Creates the neural network based on a given array of layer definitions. More...

bool	isValidNetworkDefinition (int layerCount, LayerDefinition *layerDefs)
	Validates the network definition based on a number of rules and best practices. More...

void	setLayerDefinitionDefaults (int layerCount, LayerDefinition *layerDefs)
	Returns a pointer to an array of a variable number of layer definitions. More...

LayerDefinition *	setLayerDefinitions (int layerCount,...)
	Returns a pointer to an array of a variable number of layer definitions. More...

Detailed Description

Core network library for a deep neural network.

Author: Matt Lind

Date: February 2016

Macro Definition Documentation

#define OUT_OF_RANGE -1

Function Documentation

void activateNode	(	Node *	node,
		ActFctType	actType
	)

Performs an activiation function to a specified node.

Parameters

node	Pointer to the node that is to be "activated"
actType	The type of activation function to be applied (SIGMOID/TANH/RELU)

void backPropagateLayer	(	Network *	nn,
		int	layerId
	)

Back propagates network error to hidden layer.

Backpropagating a layer means looping through all its nodes' connections, and update the errorSum attached to the TARGET node (=previous layer) of each connection i.e. when "backpropagating" layer x, then the errorSum of the nodes of layer x-1 are calculated

Parameters

nn	A pointer to the neural network
layerId	The id of the layer that is to be back propagated

void backPropagateNetwork	(	Network *	nn,
		int	targetClassification
	)

Backpropagates the output nodes' errors from output layer backwards to first layer.

The network's backpropagation proceeds in 2 steps:

CALCULATE OUTPUT NODES' ERRORS a. Calculate the errorsums in all output cells based on the targetClassification
BACKPROPAGATE EACH LAYER a. Update the nodes weights based on actual output and accumulated errorsum b. Calculate the errorsums in all TARGET cells based on errorsum in this layer (calculated in 3)

Parameters

nn	A pointer to the neural network
targetClassification	The correct/desired classification (=label) of this recognition/image

void backPropagateOutputLayer	(	Network *	nn,
		int	targetClassification
	)

Calculates the error (difference of desired classification vs actual node output) of each output node and back propagates the error in the output layer to the previous layer.

The error is calculated based on the given target classification (= image label) and is stored in each output node so that it can be backpropagated later

Parameters

nn	A pointer to the neural network
targetClassification	The correct/desired classification (=label) of this recognition/image

void calcFilterColumnIds	(	Layer *	srcLayer,
		int	srcColId,
		Layer *	tgtLayer,
		Vector *	filterColIds
	)

Returns an array of FILTER-many column ids representing a moving x*y kernel window in the target layer.

The node ids are calculated relative to the nodeId of the parent/calling feature map. The size of the moving filter/frame is defined in the parent/calling layer's filter width/height values. The size of the parent/calling feature map is bigger than the target feature map, i.e. each x'th (2nd) node, horizontally as well as vertically, is skipped. If a filter's target node would be located outside of the target feature map, -1 is assigned as an id. Later all nodes pointing to a -1 node get assigned a weight pointer to the network's "nn->nullWeight", so that they are still dereferencable. (Because NULL pointers would cause an exception/termination.)

Parameters

srcLayer	A pointer to the convolutional layer (SOURCE) that creates connections to its previous layer
srcColId	The id of the column in the convolutional (SOURCE) layer that creates the connections
tgtLayer	A pointer to the TARGET layer to which the convolutional/source layer connects to
filterColIds	A pointer to the vector which will return the list of filter ids

void calcNetworkLayer ( Layer * layer )

Calculates the output values of all nodes of a given layer.

Parameters

layer Pointer to the layer whose nodes are to be activated/calculated

double calcNodeError ( Node * thisNode )

Returns the total error of a node by adding up all the partial errors from the following layer.

To speed up back propagation the partial errors are referenced via the node's forward connections

Parameters

thisNode A pointer to the node whose (to be back propagated) error is to be calculated

void calcNodeOutput ( Node * node )

Calculates the output value of a specified node.

Calculates the vector product of a node's weights with the connections' target nodes' outputs

Parameters

node	Pointer to the node whose output is to be calculated

int calcStride	(	int	tgtWidth,
		int	filter,
		int	srcWidth
	)

Calculates the stride (number of nodes/columns that are skipped) in a convolutional kernel.

Parameters

tgtWidth	Number of columns on the x-axis (horizontally) in the TARGET (=previous) layer
filter	Number of columns/nodes on the x-axis in a filter window (

Attention: ASSSUMES WIDTH=HEIGHT!!)

Parameters

srcWidth Number of columns on the x-axis (horizontally) in the SOURCE (=this) layer

Vector* createFilterColumnIds	(	Layer *	thisLayer,
		int	columnId,
		Layer *	prevLayer
	)

Creates and returns an array of FILTER-many column ids representing a moving x*y kernel window.

The actual calculation of the target column ids takes place in a subfunction.

Parameters

thisLayer	A pointer to the convolutional layer (SOURCE) that creates connections to its previous layer
columnId	The id of the column in the source layer
prevLayer	A pointer to the PREVIOUS=TARGET layer to which the convolutional/source layer connects to

Network* createNetwork	(	int	layerCount,
		LayerDefinition *	layerDefs
	)

Creates the neural network based on a given array of layer definitions.

Creates a reserved memory block for this network based on the given layer definitions, and then initializes this memory with the respective layer/node/connection/weights structure.

Parameters

layerCount	The number of layer definitions inside the layer-definition-array (2nd param)
layerDefs	A pointer to an array of layer definitions

void feedForwardNetwork ( Network * nn )

Feeds forward (=calculating a node's output value and applying an activation function) layer by layer.

Feeds forward from 2nd=#1 layer (i.e. skips input layer) to output layer

Parameters

nn	A pointer to the NN

void feedInput	(	Network *	nn,
		Vector *	v
	)

Feeds some Vector data into the INPUT layer of the network.

Parameters

nn	A pointer to the neural network
v	A pointer to the vector holding the input values

int getColumnCount ( LayerDefinition * layerDef )

Returns the number of columns in a layer (based on a give layer definition)

Parameters

layerDef A pointer to the layer definition for this layer

Node* getColumnNode	(	Column *	column,
		int	nodeId
	)

Returns a pointer to a specific node defined by its id from a given layer.

The node is retrieved by moving a pointer from this layer's 1st node forward by id*nodeSize (it is NOT possible to retrieve a node simply via an array because the actual size of a node depends on its number of connections)

Parameters

column	A pointer to the column where this node is located in
nodeId	The id of the node that is to be returned

ByteSize getColumnSize ( LayerDefinition * layerDef )

Returns the memory (byte) size of a column based on a given layer definition.

Parameters

layerDef A pointer to a layer definition

Weight getDerivative	(	Weight	outVal,
		ActFctType	actType
	)

Returns the result of applying the given outputValue to the derivate of the activation function.

Parameters

outVal	Output value that is to be back propagated
actType	The type of activation function that was applied during feed forward (SIGMOID/TANH/RELU)

Column* getLayerColumn	(	Layer *	layer,
		int	columnId
	)

Returns a pointer to a specific column defined by its id.

The column is retrieved by moving a pointer from the layer's 1st column forward (it is NOT possible to retrieve a column simply via an array because the actual size of each column depends on its number/sizes of nodes and thus is variable

Parameters

layer	A pointer to the layer from which to get the column
columnId	The id of the column that is to be retrieved/accessed

int getLayerColumnCount ( LayerDefinition * layerDef )

Returns the number of columns in a layer.

Parameters

layerDef Pointer to a layer definition

int getLayerNodeCount ( LayerDefinition * layerDef )

Returns the number of nodes in a layer.

Parameters

layerDef Pointer to a layer definition

ByteSize getLayerSize ( LayerDefinition * layerDef )

Returns the memory (byte) size of a specific layer based on a given layer definition.

Each layer's memory size may be different due to a different number of nodes and connections For FEED FORWARD (e.g. OUTPUT) layers, full connectivity is assumed (i.e. each node links to ALL nodes in the previous layer)

Parameters

layerDef A pointer to a layer definition

ByteSize getLayerWeightBlockSize ( LayerDefinition * layerDef )

Returns the memory (byte) size of the weights block for a specific layer.

Each layer's number of weights may be different due to a different number of connections For FEED FORWARD (HIDDEN and OUTPUT) layers, full connectivity is assumed, CONV layers e.g. share weights.

Parameters

layerDef A pointer to a layer definition

int getLayerWeightCount ( LayerDefinition * layerDef )

Returns the number of weights for a layer (based on a given layer definition)

Parameters

layerDef A pointer to the layer definition

int getNetworkClassification ( Network * nn )

Returns the network's classification of the input image by choosing the node with the hightest output.

Parameters

nn	A pointer to the neural network

Layer* getNetworkLayer	(	Network *	nn,
		int	layerId
	)

Returns a pointer to a specific layer defined by its id from the network.

The layer is retrieved by moving a pointer from the network's 1st layer forward by layerId*layerSize (it is NOT possible to retrieve a layer simply via an array because the actual size of EACH layer depends on its number/sizes of nodes)

Parameters

nn	A pointer to the NN
layerId	The id of the layer that is to be returned

Node* getNetworkNode	(	Layer *	layer,
		int	columnId,
		int	nodeId
	)

Returns a pointer to a specific node defined by its layer, column and node id.

Parameters

layer	A pointer to a network layer
columnId	The id of the column inside this layer
nodeId	The id of the node inside this column

ByteSize getNetworkSize	(	int	layerCount,
		LayerDefinition *	layerDefs
	)

Returns the memory size of the network based on an array of layer definitions and weightBlockSize.

Parameters

layerCount	Number of defined layers for this network
layerDefs	Array of layer definitions

number of columns = width * height number of nodes per column = depth number of nodes = width * height * depth number of connections = filter * depth of previous layer * number of nodes number of weights = filter * depth of previous layer * depth of this layer

ByteSize getNetworkWeightBlockSize	(	int	layerCount,
		LayerDefinition *	layerDefs
	)

Returns the memory size of the network's weights block based on a given array of layer definitions.

Each layer's number of weights may be different due to a different number of nodes & connections The weight block is a block of memory that is located inside the network object, AFTER the layers.

Parameters

layerCount	The number of layers in the network
layerDefs	A pointer to an array of layer definitions

int getNodeBackwardConnectionCount ( LayerDefinition * layerDef )

Returns the number of backward connections of a NODE (not of a layer)

For FEED FORWARD (HIDDEN and OUTPUT) layers, full connectivity is assumed (each node links to ALL nodes in the previous layer)

Parameters

layerDef Pointer to a layer definition

int getNodeForwardConnectionCount ( LayerDefinition * layerDef )

Returns the number of forward connections of a NODE (not of a layer)

Attention: The number of FORWARD connections in one layer is NOT the same as the number of BACKWARD connections in the following layer!!

Parameters

layerDef A pointer to the layer definition

ByteSize getNodeSize ( LayerDefinition * layerDef )

Returns the memory (byte) size of a node based on a given layer definition.

Each layer's nodes' memory size may be different due to a different number of connections For FEED FORWARD (HIDDEN and OUTPUT) layers, full connectivity is assumed (each node links to ALL nodes in the previous layer)

Parameters

layerDef Pointer to a layer definition

void initNetwork	(	Network *	nn,
		int	layerCount,
		LayerDefinition *	layerDefs
	)

void initNetworkBackwardConnectionsConvNode	(	Node *	node,
		int	srcLevel,
		Weight *	srcLayerWeightPtr,
		Layer *	targetLayer,
		Vector *	filterColIds,
		Weight *	nullWeight
	)

Initializes a single convolutional node by setting its connections weights pointers.

Each convolutional node has connections to a filter/kernel window of nodes in the previous layer.

Parameters

node	A pointer to the convolutional node whose connections are to be set/initialized
srcLevel	The node's level inside the column. Needed to calculate the position of the respective weight.
srcLayerWeightPtr	A pointer to the weight block of this (=convolutional node's) layer
targetLayer	A pointer to the target layer to which this convolutional node shall connect to
filterColIds	A vector of indeces/positions of the target columns/nodes that this node connects to
nullWeight	A pointer to a weight that is used (1) to initialize or (2) to link "dead" connections

The following describes the logic/algorithm for calculating the weights' position

use WEIGHT MATRIX (of size filter * filter) for LEVEL 1 establish connections from source node to all nodes on LEVEL 1 inside the TARGET FILTER using WM1 use WEIGHT MATRIX of LEVEL 2 establish connections from source node to all nodes on LEVEL 2 inside the TARGET FILTER using WM2 same for all levels of TARGET LAYER

move source node to next node (= next node/level in SAME COLUMN!)

do same as above, again from LEVEL1-n using WM1-n

weights pointer position = (srcLevel * tgtDepth * filterSize) + (tgtLevel * filterSizer) + filterColId

move the filter using the SAME WEIGHT MATRIX across the target layer then move 1 level down in the TARGET layer

void initNetworkBackwardConnectionsFCNode	(	Node *	thisNode,
		Layer *	prevLayer,
		Weight *	nodeWeightPtr
	)

Initializes a node of a normal, fully connected node.

Creates connections with pointers towards all nodes of the previous layer (=fully connected)

Attention: The node's bias weight is not initialized here but together with the weights

Parameters

thisNode	A pointer to the node whose connections are to be added/initialized
prevLayer	A pointer to the PREVIOUS layer which this node will connect to
nodeWeightPtr	A pointer to the memory block that is to store the weights of this node

void initNetworkColumns	(	Network *	nn,
		int	layerId
	)

Initializes a network layer's column/node structure and sets detault values.

A column is a vector of nodes. The number of nodes in a colum is defined as the "DEPTH" (or number of feature maps) For non-convolutional layers the DEPTH (i.e. number of nodes in a column) is 1.

Parameters

nn	A pointer the network
layerId	The id of the layer whose column are to be initialized

void initNetworkForwardConnections	(	Network *	nn,
		int	layerId
	)

void initNetworkForwardConnectionsAnyNode	(	Node *	thisNode,
		Layer *	nextLayer
	)

void initNetworkLayer	(	Network *	nn,
		int	layerId,
		LayerDefinition *	layerDefs
	)

Initializes a network layer by creating the column/node structure and sets detault values.

Parameters

nn	A pointer to the neural network
layerId	The id of the layer which is to be initialized
layerDefs	Pointer to an array of layer definitions

void initNetworkNodes	(	Network *	nn,
		int	layerId,
		int	columnId
	)

Initializes the nodes in a given network column.

Creates the column/node structure inside the network's respective memory block. Connections will be initialized with a NULL pointer and a default ->nullWeight pointer (for null weights I don't use NULL to avoid exceptions if it is (mistakenly?) dereferenced)

Parameters

nn	A pointer to the network
layerId	The index of the layer whose column=nodes are to be initialized
columnId	The index of the column whose nodes are to be initialized

void initNetworkWeights ( Network * nn )

bool isValidNetworkDefinition	(	int	layerCount,
		LayerDefinition *	layerDefs
	)

Validates the network definition based on a number of rules and best practices.

Checks whether the provided layer definitions define a proper/feasible a neural network

Parameters

layerCount	Number of defined layers for this network
layerDefs	A pointer to an array of layer definitions

void setLayerDefinitionDefaults	(	int	layerCount,
		LayerDefinition *	layerDefs
	)

Returns a pointer to an array of a variable number of layer definitions.

Parameters

layerCount	Number of layers of the network
layerDefs	Variabe number of layer definition objects

LayerDefinition* setLayerDefinitions	(	int	layerCount,
			...
	)

Returns a pointer to an array of a variable number of layer definitions.

Parameters

layerCount	Number of layers of the network
...	Variabe number of layer definition objects

void setNetworkDefaults	(	Network *	nn,
		int	layerCount,
		LayerDefinition *	layerDefs,
		ByteSize	netSize
	)

Sets the network's default values for size, layerCount,.

Theses values are needed to "navigate" inside the network object

Parameters

nn	A pointer to the neural network
layerCount	The number of layers of this network
layerDefs	A pointer to an array of the layer definitions for this network
netSize	Total memory size of the network

void setNetworkNodeDefaults	(	Layer *	thisLayer,
		Column *	column,
		Node *	node,
		Weight *	nullWeight
	)

Sets default values for a node during its initialization.

Parameters

thisLayer	A pointer to the layer in which the node is located
column	A pointer to the column in which the node is located
node	A pointer to the node whose values are to be (re)set
nullWeight	A pointer to the network's null weight

void updateNodeWeights	(	Node *	updateNode,
		double	learningRate
	)

Updates a node's weights based on given learning rate.

The accumulated error (difference between desired output and actual output) of this node must have been calculated before and attached to the node (= ->errorSum)

Parameters

updateNode	A pointer to the node whose weights are to be updated
learningRate	The factor with which errors are applied to weights

Macros

Functions

Detailed Description

Macro Definition Documentation

Function Documentation