/**
 * A node in a neural network. Each node has a state
 * (total input, output, and their respectively derivatives) which changes
 * after every forward and back propagation run.
 */ export class Node { id : string ; /** List of input links. */ inputLinks : Link [ ] = [ ] ; bias = 0.1 ; /** List of output links. */ outputs : Link [ ] = [ ] ; totalInput : number ; output : number ; layer : number ; trueLearningRate = 0 ; /** Error derivative with respect to this node's output. */ outputDer = 0 ; /** Error derivative with respect to this node's total input. */ inputDer = 0 ; /**
     * Accumulated error derivative with respect to this node's total input since
     * the last update. This derivative equals dE/db where b is the node's
     * bias term.
     */ accInputDer = 0 ; /**
     * Number of accumulated err. derivatives with respect to the total input
     * since the last update.
     */ numAccumulatedDers = 0 ; /** Activation function that takes total input and returns node's output */ activation : ActivationFunction ; /**
     * Creates a new node with the provided id and activation function.
     */ constructor ( id , activation , initZero ? ) { this . id = id ; this . activation = activation ; if ( initZero ) { this . bias = 0 ; } } /** Recomputes the node's output and returns it. */ updateOutput ( ) { this . totalInput = this . bias ; for ( let j = 0 ; j < this . inputLinks . length ; j ++ ) { let link = this . inputLinks [ j ] ; this . totalInput += link . weight * link . source . output ; } this . output = this . activation . output ( this . totalInput ) ; return this . output ; } } /**
 * An error function and its derivative.
 */ export interface ErrorFunction { error : ( output : number , target : number ) => number ; der : ( output : number , target : number ) => number ; } /** A node's activation function and its derivative. */ export interface ActivationFunction { output : ( input : number ) => number ; der : ( input : number ) => number ; } /** Function that computes a penalty cost for a given weight in the network. */ export interface RegularizationFunction { output : ( weight : number ) => number ; der : ( weight : number ) => number ; } /** Built-in error functions */ export class Errors { public static SQUARE : ErrorFunction = { error : ( output , target ) => 0.5 * Math . pow ( output - target , 2 ) , der : ( output , target ) => output - target } ; } /** Polyfill for TANH */ ( Math as any ) . tanh = ( Math as any ) . tanh || function ( x ) { if ( x === Infinity ) { return 1 ; } else if ( x === - Infinity ) { return - 1 ; } else { let e2x = Math . exp ( 2 * x ) ; return ( e2x - 1 ) / ( e2x + 1 ) ; } } ; /** Built-in activation functions */ export class Activations { public static TANH : ActivationFunction = { output : x => ( Math as any ) . tanh ( x ) , der : x => { let output = Activations . TANH . output ( x ) ; return 1 - output * output ; } } ; public static RELU : ActivationFunction = { output : x => Math . max ( 0 , x ) , der : x => x <= 0 ? 0 : 1 } ; public static SIGMOID : ActivationFunction = { output : x => 1 / ( 1 + Math . exp ( - x ) ) , der : x => { let output = Activations . SIGMOID . output ( x ) ; return output * ( 1 - output ) ; } } ; public static LINEAR : ActivationFunction = { output : x => x , der : x => 1 } ; public static SINX : ActivationFunction = { output : x => Math . sin ( x ) , der : x => Math . cos ( x ) } ; } /** Build-in regularization functions */ export class RegularizationFunction { public static L1 : RegularizationFunction = { output : w => Math . abs ( w ) , der : w => w < 0 ? - 1 : ( w > 0 ? 1 : 0 ) } ; public static L2 : RegularizationFunction = { output : w => 0.5 * w * w , der : w => w } ; } /**
 * A link in a neural network. Each link has a weight and a source and
 * destination node. Also it has an internal state (error derivative
 * with respect to a particular input) which gets updated after
 * a run of back propagation.
 */ export class Link { id : string ; source : Node ; dest : Node ; weight = Math . random ( ) - 0.5 ; isDead = false ; /** Error derivative with respect to this weight. */ errorDer = 0 ; /** Accumulated error derivative since the last update. */ accErrorDer = 0 ; /** Number of accumulated derivatives since the last update. */ numAccumulatedDers = 0 ; regularization : RegularizationFunction ; trueLearningRate = 0 ; isResidual : boolean = false ; param source param dest param regularization constructor ( source , dest , regularization , initZero ? ) { this . id = source . id + "-" + dest . id ; this . source = source ; this . dest = dest ; this . regularization = regularization ; if ( initZero ) { this . weight = 0 ; } } } param networkShape param activation param outputActivation param regularization param inputIds export function buildNetwork ( networkShape , activation , outputActivation , regularization , inputIds , initZero ? ) { let numLayers = networkShape . length ; let id = 1 ; /** List of layers, with each layer being a list of nodes. */ let network = [ ] ; for ( let layerIdx = 0 ; layerIdx < numLayers ; layerIdx ++ ) { let isOutputLayer = layerIdx === numLayers - 1 ; let isInputLayer = layerIdx === 0 ; let currentLayer = [ ] ; network . push ( currentLayer ) ; let numNodes = networkShape [ layerIdx ] ; for ( let i = 0 ; i < numNodes ; i ++ ) { let nodeId = id . toString ( ) ; if ( isInputLayer ) { nodeId = inputIds [ i ] ; } else { id ++ ; } let node = new Node ( nodeId , isOutputLayer ? outputActivation : activation , initZero ) ; node . layer = layerIdx ; currentLayer . push ( node ) ; if ( layerIdx >= 1 ) { for ( let j = 0 ; j < network [ layerIdx - 1 ] . length ; j ++ ) { let prevNode = network [ layerIdx - 1 ] [ j ] ; let link = new Link ( prevNode , node , regularization , initZero ) ; prevNode . outputs . push ( link ) ; node . inputLinks . push ( link ) ; } } } } return network ; } param network param inputs return export function forwardProp ( network , inputs ) { let inputLayer = network [ 0 ] ; if ( inputs . length !== inputLayer . length ) { throw new Error ( "The number of inputs must match the number of nodes in" + " the input layer" ) ; } for ( let i = 0 ; i < inputLayer . length ; i ++ ) { let node = inputLayer [ i ] ; node . output = inputs [ i ] ; } for ( let layerIdx = 1 ; layerIdx < network . length ; layerIdx ++ ) { let currentLayer = network [ layerIdx ] ; for ( let i = 0 ; i < currentLayer . length ; i ++ ) { let node = currentLayer [ i ] ; node . updateOutput ( ) ; } } return network [ network . length - 1 ] [ 0 ] . output ; } /**
 * Runs a backward propagation using the provided target and the
 * computed output of the previous call to forward propagation.
 * This method modifies the internal state of the network - the error
 * derivatives with respect to each node, and each weight
 * in the network.
 */ export function backProp ( network , target , errorFunc ) { let outputNode = network [ network . length - 1 ] [ 0 ] ; outputNode . outputDer = errorFunc . der ( outputNode . output , target ) ; for ( let layerIdx = network . length - 1 ; layerIdx >= 1 ; layerIdx -- ) { let currentLayer = network [ layerIdx ] ; for ( let i = 0 ; i < currentLayer . length ; i ++ ) { let node = currentLayer [ i ] ; node . inputDer = node . outputDer * node . activation . der ( node . totalInput ) ; node . accInputDer += node . inputDer ; node . numAccumulatedDers ++ ; } for ( let i = 0 ; i < currentLayer . length ; i ++ ) { let node = currentLayer [ i ] ; for ( let j = 0 ; j < node . inputLinks . length ; j ++ ) { let link = node . inputLinks [ j ] ; if ( link . isDead ) { continue ; } link . errorDer = node . inputDer * link . source . output ; link . accErrorDer += link . errorDer ; link . numAccumulatedDers ++ ; } } if ( layerIdx === 1 ) { continue ; } let prevLayer = network [ layerIdx - 1 ] ; for ( let i = 0 ; i < prevLayer . length ; i ++ ) { let node = prevLayer [ i ] ; node . outputDer = 0 ; for ( let j = 0 ; j < node . outputs . length ; j ++ ) { let output = node . outputs [ j ] ; node . outputDer += output . weight * output . dest . inputDer ; } } } } /**
 * Updates the weights of the network using the previously accumulated error
 * derivatives.
 */ export function updateWeights ( network , learningRate , regularizationRate ) { for ( let layerIdx = 1 ; layerIdx < network . length ; layerIdx ++ ) { let currentLayer = network [ layerIdx ] ; for ( let i = 0 ; i < currentLayer . length ; i ++ ) { let node = currentLayer [ i ] ; if ( node . numAccumulatedDers > 0 ) { node . trueLearningRate = node . accInputDer / node . numAccumulatedDers ; node . bias -= learningRate * node . trueLearningRate ; node . accInputDer = 0 ; node . numAccumulatedDers = 0 ; } for ( let j = 0 ; j < node . inputLinks . length ; j ++ ) { let link = node . inputLinks [ j ] ; if ( link . isDead ) { continue ; } let regulDer = link . regularization ? link . regularization . der ( link . weight ) : 0 ; if ( link . numAccumulatedDers > 0 ) { link . trueLearningRate = link . accErrorDer / link . numAccumulatedDers ; link . weight = link . weight - learningRate * link . trueLearningRate ; let newLinkWeight = link . weight - ( learningRate * regularizationRate ) * regulDer ; if ( link . regularization === RegularizationFunction . L1 && link . weight * newLinkWeight < 0 ) { link . weight = 0 ; link . isDead = true ; } else { link . weight = newLinkWeight ; } link . accErrorDer = 0 ; link . numAccumulatedDers = 0 ; } } } } } /** Iterates over every node in the network/ */ export function forEachNode ( network , ignoreInputs , accessor ) { for ( let layerIdx = ignoreInputs ? 1 : 0 ; layerIdx < network . length ; layerIdx ++ ) { let currentLayer = network [ layerIdx ] ; for ( let i = 0 ; i < currentLayer . length ; i ++ ) { let node = currentLayer [ i ] ; accessor ( node ) ; } } } /** Returns the output node in the network. */ export function getOutputNode ( network ) { return network [ network . length - 1 ] [ 0 ] ; }