import { int , double } from "./main" ; import { InputLayerConfig , LayerConfig , OutputLayerConfig } from "./Presets" ; import { makeArray } from "./Util" ; import { Configuration , TrainingData , TrainingDataEx } from "./Configuration" ; /**
 * Simple implementation of a neural network (multilayer perceptron)
 *
 * Uses stochastic gradient descent with squared error as the loss function
 */ export namespace Net { /** tangens hyperbolicus polyfill */ const tanh = function ( x ) { if ( x === Infinity ) { return 1 ; } else if ( x === - Infinity ) { return - 1 ; } else { var y = Math . exp ( 2 * x ) ; return ( y - 1 ) / ( y + 1 ) ; } } ; /** an activation function (non-linearity) and its derivative */ interface ActivationFunction { f : ( x : double ) => double ; df : ( x : double ) => double ; } /** list of known activation functions */ export var NonLinearities = { sigmoid : { f : x => 1 / ( 1 + Math . exp ( - x ) ) , df : x => { x = 1 / ( 1 + Math . exp ( - x ) ) ; return x * ( 1 - x ) ; } } , tanh : { f : x => tanh ( x ) , df : x => { x = tanh ( x ) ; return 1 - x * x ; } } , linear : { f : x => x , df : x => 1 } , relu : { f : x => Math . max ( x , 0 ) , df : x => ( x <= 0 ? 0 : 1 ) } , lrelu : { f : x => ( x > 0 ? x : x * 0.1 ) , df : x => ( x <= 0 ? 0 : 1 ) } , "threshold (≥ 0)" : { f : x => ( x >= 0 ? 1 : 0 ) , df : x => 1 } } ; /**
     * A training method for a neural network.
     *
     * optionally returns WeightsSteps for displaying of intermediate data in the NetworkVisualization
     *
     * if trainSingle is null, only batch training is possible
     */ export interface TrainingMethod { trainAll : ( net : NeuralNet , data : TrainingDataEx [ ] ) => WeightsStep [ ] | null ; trainSingle : ( ( net : NeuralNet , data : TrainingData ) => WeightsStep | null ) | null ; } /** list of training methods for each Configuration#type */ export const trainingMethods = { nn : { "Batch Training" : { trainAll : ( net , data ) => net . trainAll ( data , false , false ) , trainSingle : null } , "Online Training" : { trainAll : ( net , data ) => net . trainAll ( data , true , false ) , trainSingle : ( net , data ) => net . train ( data , true , false ) } , "Time Delayed Training" : { trainAll : ( net , data ) => net . trainAll ( data , true , false ) , trainSingle : ( net , data ) => net . train ( data , true , false ) } } , /** A Perceptron is a special type of NeuralNet that has an input layer with 3 neurons (incl. bias), an output layer with 1 neuron, and no hidden layers */ perceptron : { "Rosenblatt Perceptron" : { trainAll ( net , data ) { return data . map ( val => this . trainSingle ! ( net , val ) ) as WeightsStep [ ] ; } , trainSingle : ( net , val ) => { net . setInputsAndCalculate ( val . input ) ; const outp = net . outputs [ 0 ] ; outp . targetOutput = val . output [ 0 ] ; const err = outp . targetOutput - outp . output ; for ( const conn of outp . inputs ) { conn . weight += net . learnRate * err * conn . inp . output ; } var weights = net . connections . map ( conn => conn . weight ) ; return { weights , dataPoint : val } ; } } , "Batch Perceptron" : { trainAll : ( net , data ) => net . trainAll ( data , false , true ) , trainSingle : null } , /** averaged perceptron (from http://ciml.info/dl/v0_8/ciml-v0_8-ch03.pdf , p.48) */ "Averaged Perceptron" : { trainAll : ( net , data ) => { if ( ! ( net as any ) . tmpStore ) ( net as any ) . tmpStore = { c : 1 , w : net . connections . map ( c => c . weight ) , u : net . connections . map ( c => 0 ) } ; const vars = ( net as any ) . tmpStore ; const storeWeightSteps = true ; if ( net . layers . length !== 2 || net . outputs . length !== 1 || net . outputs [ 0 ] . activation !== "threshold (≥ 0)" ) throw Error ( "can only be used for single perceptron" ) ; if ( storeWeightSteps ) var weights = [ ] ; for ( const val of data ) { const y = val . output [ 0 ] ? 1 : - 1 ; const x = val . input . concat ( [ 1 ] ) ; let yReal = 0 ; for ( let i = 0 ; i < x . length ; i ++ ) yReal += x [ i ] * vars . w [ i ] ; if ( y * yReal <= 0 ) { for ( let i = 0 ; i < x . length ; i ++ ) { vars . w [ i ] += net . learnRate * y * x [ i ] ; vars . u [ i ] += net . learnRate * y * vars . c * x [ i ] ; } if ( storeWeightSteps ) weights ! . push ( { dataPoint : val , weights : x . map ( ( x , i ) => vars . w [ i ] - vars . u [ i ] / vars . c ) } ) ; } ++ vars . c ; } net . connections . forEach ( ( conn , i ) => ( conn . weight = vars . w [ i ] - vars . u [ i ] / vars . c ) ) ; return weights ! ; } , trainSingle : null } , /** from http://www.jmlr.org/papers/volume3/crammer03a/crammer03a.pdf , page 965
             * equivalent to Rosenblatt Perceptron except for how G is defined (see pdf)
             */ "Binary MIRA" : { trainAll ( net , data ) { return data . map ( d => this . trainSingle ! ( net , d ) ! ) ; } , trainSingle : ( net , val ) => { net . setInputsAndCalculate ( val . input ) ; const outp = net . outputs [ 0 ] ; const x = val . input ; const y = val . output [ 0 ] == 1 ? 1 : - 1 ; const yGot = net . outputs [ 0 ] . output == 1 ? 1 : - 1 ; const G = ( x ) => Math . min ( Math . max ( 0 , x ) , 1 ) ; const err = G ( ( - y * net . outputs [ 0 ] . weightedInputs ) / x . reduce ( ( a , b ) => a + b * b , 0 ) ) ; for ( const conn of outp . inputs ) { conn . weight += net . learnRate * err * y * conn . inp . output ; } var weights = net . connections . map ( conn => conn . weight ) ; return { weights , dataPoint : val } ; } } } } ; /**
     * intermediate result of a single data point training step and the resulting weights vector
     *
     * used to visualize the weights vector in the perceptron demo
     */ export interface WeightsStep { dataPoint : TrainingData | null ; weights : number [ ] ; } /**
     * back propagation code adapted from https://de.wikipedia.org/wiki/Backpropagation
     */ export class NeuralNet { /** layers including input, hidden, and output */ layers : Neuron [ ] [ ] = [ ] ; /** bias input neurons */ biases : InputNeuron [ ] = [ ] ; /** actual input neurons */ inputs : InputNeuron [ ] ; outputs : OutputNeuron [ ] ; isTDNN : boolean = false ; /** a flat list of all the neuron connections */ connections : NeuronConnection [ ] = [ ] ; constructor ( input , hidden , output , public learnRate , timeDelayed ? , startWeight = ( ) => Math . random ( ) - 0.5 , public startWeights ? ) { let nid = 0 ; this . inputs = makeArray ( input . neuronCount , i => new InputNeuron ( nid ++ , i , input . names [ i ] ) ) ; if ( timeDelayed ) this . isTDNN = true ; this . layers . push ( this . inputs . slice ( ) ) ; for ( var layer of hidden ) { this . layers . push ( makeArray ( layer . neuronCount , i => timeDelayed != undefined ? new TimeDelayedNeuron ( layer . activation , nid ++ , i , timeDelayed [ hidden . indexOf ( layer ) ] ) : new Neuron ( layer . activation , nid ++ , i ) ) ) ; } this . outputs = makeArray ( output . neuronCount , i => new OutputNeuron ( output . activation , nid ++ , i , output . names [ i ] ) ) ; this . layers . push ( this . outputs ) ; for ( let i = 0 ; i < this . layers . length - 1 ; i ++ ) { const inLayer = this . layers [ i ] ; const outLayer = this . layers [ i + 1 ] ; if ( timeDelayed == undefined ) inLayer . push ( new InputNeuron ( nid ++ , - 1 , "Bias" , 1 ) ) ; if ( timeDelayed != undefined && outLayer [ 0 ] instanceof OutputNeuron ) { for ( let inputI = 0 ; inputI < inLayer . length ; inputI ++ ) { const input = inLayer [ inputI ] ; var conn = new Net . NeuronConnection ( input , outLayer [ inputI ] ) ; input . outputs . push ( conn ) ; outLayer [ inputI ] . inputs . push ( conn ) ; this . connections . push ( conn ) ; } } else { for ( const input of inLayer ) for ( const output of outLayer ) { var conn = new Net . NeuronConnection ( input , output ) ; input . outputs . push ( conn ) ; output . inputs . push ( conn ) ; this . connections . push ( conn ) ; } } if ( timeDelayed == undefined ) this . biases [ i ] = inLayer . pop ( ) as InputNeuron ; } if ( ! this . startWeights ) { this . startWeights = this . connections . map ( c => ( c . weight = startWeight ( ) ) ) ; } else this . startWeights . forEach ( ( w , i ) => ( this . connections [ i ] . weight = w ) ) ; } setInputsAndCalculate ( inputVals ) { for ( let i = 0 ; i < this . inputs . length ; i ++ ) this . inputs [ i ] . output = inputVals [ i ] ; for ( const layer of this . layers . slice ( 1 ) ) for ( const neuron of layer ) neuron . calculateOutput ( ) ; } initTDNN ( inputVals ) { for ( let i = 0 ; i < this . inputs . length ; i ++ ) this . inputs [ i ] . outputVector = inputVals [ i ] ; for ( const layer of this . layers . slice ( 1 ) ) for ( const neuron of layer ) { for ( const conn of neuron . inputs ) { for ( var i = 0 ; i <= conn . inp . outputVector . length - neuron . timeDelayed ; i ++ ) { neuron . outputVector [ i ] = 0 ; } } } } setInputVectorsAndCalculate ( inputVals ) { for ( let i = 0 ; i < this . inputs . length ; i ++ ) { this . inputs [ i ] . outputVector = inputVals [ i ] ; for ( let j = 0 ; j < inputVals . length ; j ++ ) { if ( inputVals [ i ] [ j ] > 1 ) this . inputs [ i ] . outputVector [ j ] = inputVals [ i ] [ j ] /= 255 ; } } for ( let layer of this . layers . slice ( 1 , this . layers . length - 1 ) ) { for ( let neuron of layer ) neuron . calculateOutput ( ) ; } const lastHiddenLayer = this . layers [ this . layers . length - 2 ] ; const outputLayer = this . outputs ; for ( let neuron of outputLayer ) { neuron . calculateOutputTDNN ( lastHiddenLayer [ outputLayer . indexOf ( neuron ) ] . outputVector ) ; } this . outputs . map ( output => output . output ) ; return this . outputs . map ( output => output . output ) ; } getOutput ( inputVals ) { this . setInputsAndCalculate ( inputVals ) ; return this . outputs . map ( output => output . output ) ; } /** get root-mean-square error */ getLoss ( expectedOutput ) { let sum = 0 ; for ( let i = 0 ; i < this . outputs . length ; i ++ ) { const neuron = this . outputs [ i ] ; sum += Math . pow ( neuron . output - expectedOutput [ i ] , 2 ) ; } return Math . sqrt ( sum / this . outputs . length ) ; } /** if individual is true, train individually, else batch train */ trainAll ( data , individual , storeWeightSteps ) { let weights = null ; if ( storeWeightSteps ) weights = [ ] ; if ( ! individual ) for ( const conn of this . connections ) conn . zeroDeltaWeight ( ) ; for ( const val of data ) { const step = this . train ( val , individual , storeWeightSteps ) ; if ( weights ) weights . push ( step ! ) ; } if ( ! individual ) for ( const conn of this . connections ) conn . flushDeltaWeight ( ) ; return weights ; } /** if flush is false, only calculate deltas but don't reset or add them */ train ( val , flush = true , storeWeightSteps ) { if ( this . isTDNN ) this . setInputVectorsAndCalculate ( val . inputVector ! ) ; else this . setInputsAndCalculate ( val . input ) ; for ( let i = 0 ; i < this . outputs . length ; i ++ ) { this . outputs [ i ] . targetOutput = val . output [ i ] ; } for ( let i = this . layers . length - 1 ; i > 0 ; i -- ) { for ( const neuron of this . layers [ i ] ) { neuron . calculateError ( ) ; for ( const conn of neuron . inputs ) { if ( flush ) conn . zeroDeltaWeight ( ) ; conn . addDeltaWeight ( this . learnRate ) ; } } } var w = storeWeightSteps ? { weights : this . connections . map ( conn => conn . weight + conn . deltaWeight ) , dataPoint : val } : null ; if ( flush ) for ( const conn of this . connections ) conn . flushDeltaWeight ( ) ; return w ; } } /** a weighted connection between two neurons */ export class NeuronConnection { public weight = 0 ; public weightVector ? : number [ ] ; /** cached delta weight for training */ deltaWeight = NaN ; deltaWeightVector : number [ ] = [ ] ; momentum = 0.9 ; velocityVector : number [ ] = [ ] ; constructor ( /**Input neuron of this connection */ public inp , /**Output neuron of this connection */ public out ) { } zeroDeltaWeight ( ) { this . deltaWeight = 0 ; this . deltaWeightVector = [ ] ; } addDeltaWeight ( learnRate ) { if ( this . out instanceof TimeDelayedNeuron ) { var tmp = 0 ; for ( var outputNeuronIndex = 0 ; outputNeuronIndex < this . out . outputVector ! . length ; outputNeuronIndex ++ ) { for ( var time = 0 ; time < this . out . timeDelayed ; time ++ ) { var tmp1 = 0 ; if ( this . deltaWeightVector [ time ] == null ) this . deltaWeightVector [ time ] = 0 ; if ( this . velocityVector [ time ] == null ) this . velocityVector [ time ] = 0 ; tmp1 = learnRate * this . out . errorVector [ outputNeuronIndex ] * this . inp . outputVector ! [ outputNeuronIndex + time ] ; this . deltaWeightVector [ time ] += tmp1 ; } } for ( var time = 0 ; time < this . out . timeDelayed ; time ++ ) { this . deltaWeightVector [ time ] /= this . out . outputVector ! . length ; this . velocityVector [ time ] = this . momentum * this . velocityVector [ time ] + this . deltaWeightVector [ time ] ; } } else if ( this . inp instanceof TimeDelayedNeuron ) { for ( let time = 0 ; time < this . weightVector ! . length ; time ++ ) { var tmp = 0 ; tmp = learnRate * this . out . error * this . inp . outputVector ! [ time ] ; if ( this . deltaWeightVector [ time ] == null ) this . deltaWeightVector [ time ] = 0 ; if ( this . velocityVector [ time ] == null ) this . velocityVector [ time ] = 0 ; this . deltaWeightVector [ time ] = tmp ; this . velocityVector [ time ] = this . momentum * this . velocityVector [ time ] + this . deltaWeightVector [ time ] ; } } else this . deltaWeight += learnRate * this . out . error * this . inp . output ; } flushDeltaWeight ( ) { if ( this . weightVector != undefined ) { for ( var time = 0 ; time < this . out . timeDelayed ; time ++ ) { this . weightVector ! [ time ] += this . velocityVector [ time ] ; } this . deltaWeightVector = [ ] ; } else { this . weight += this . deltaWeight ; this . deltaWeight = NaN ; } } } /** a single Neuron / Perceptron */ export class Neuron { public inputs : NeuronConnection [ ] = [ ] ; public outputs : NeuronConnection [ ] = [ ] ; /** weighted sum of inputs without activation function */ public weightedInputs = 0 ; public output = 0 ; public error = 0 ; /** Variable for TDNN */ public weightedInputsVector : double [ ] = [ ] ; public errorVector : number [ ] = [ ] ; public outputVector : double [ ] = [ ] ; public timeDelayed : int = 0 ; constructor ( public activation , public id , public layerIndex ) { } calculateWeightedInputs ( ) { this . weightedInputs = 0 ; for ( const conn of this . inputs ) { this . weightedInputs += conn . inp . output * conn . weight ; } } calculateOutput ( ) { this . calculateWeightedInputs ( ) ; this . output = NonLinearities [ this . activation ] . f ( this . weightedInputs ) ; } calculateOutputTDNN ( inputVals ) { this . timeDelayed = inputVals . length ; this . weightedInputs = 0 ; for ( const conn of this . inputs ) { if ( conn . weightVector == undefined ) { conn . weightVector = [ ] ; for ( let i = 0 ; i < inputVals . length ; i ++ ) conn . weightVector . push ( Math . random ( ) - 0.5 ) ; } for ( let i = 0 ; i < inputVals . length ; i ++ ) this . weightedInputs += conn . weightVector ! [ i ] * inputVals [ i ] ; } this . output = NonLinearities [ this . activation ] . f ( this . weightedInputs ) ; } calculateError ( ) { var δ = 0 ; for ( const output of this . outputs ) { δ += output . out . error * output . weight ; } this . error = δ * NonLinearities [ this . activation ] . df ( this . weightedInputs ) ; } } /** an input neuron (no inputs, fixed output value, can't be trained) */ export class InputNeuron extends Neuron { constant : boolean = false ; constructor ( id , layerIndex , public name , constantOutput ? ) { super ( null ! , id , layerIndex ) ; if ( constantOutput !== undefined ) { this . output = constantOutput ; this . constant = true ; } } calculateOutput ( ) { throw Error ( "input neuron" ) ; } calculateWeightedInputs ( ) { throw Error ( "input neuron" ) ; } calculateError ( ) { throw Error ( "input neuron" ) ; } } /** an output neuron (error calculated via target output */ export class OutputNeuron extends Neuron { targetOutput : double = 0 ; constructor ( public activation , id , layerIndex , public name ) { super ( activation , id , layerIndex ) ; } calculateError ( ) { this . error = NonLinearities [ this . activation ] . df ( this . weightedInputs ) * ( this . targetOutput - this . output ) ; } } export class TimeDelayedNeuron extends Neuron { constructor ( public activation , public id , public layerIndex , public timeDelayed ) { super ( activation , id , layerIndex ) ; } calculateWeightedInputs ( ) { this . weightedInputsVector = [ ] ; for ( const conn of this . inputs ) { if ( ! conn . weightVector ) { console . log ( "Create weightVector" ) ; conn . weightVector = [ ] ; for ( let i = 0 ; i < this . timeDelayed ; i ++ ) if ( conn . weightVector [ i ] == null ) conn . weightVector [ i ] = Math . random ( ) - 0.5 ; } for ( var i = 0 ; i <= conn . inp . outputVector . length - this . timeDelayed ; i ++ ) { if ( this . weightedInputsVector [ i ] == null ) this . weightedInputsVector [ i ] = 0 ; for ( var j = i ; j < i + this . timeDelayed ; j ++ ) { this . weightedInputsVector [ i ] += conn . inp . outputVector [ j ] * conn . weightVector ! [ j - i ] ; } } } } calculateOutput ( ) { this . calculateWeightedInputs ( ) ; for ( var i = 0 ; i < this . weightedInputsVector . length ; i ++ ) { this . outputVector [ i ] = NonLinearities [ this . activation ] . f ( this . weightedInputsVector [ i ] ) ; } } calculateError ( ) { var δ = 0 ; if ( this . outputs [ 0 ] . out instanceof OutputNeuron ) { for ( const output of this . outputs ) { for ( var weightIndex = 0 ; weightIndex < output . weightVector ! . length ; weightIndex ++ ) { var tmpδ = 0 ; tmpδ += output . out . error * output . weightVector ! [ weightIndex ] ; this . errorVector [ weightIndex ] = tmpδ * NonLinearities [ this . activation ] . df ( this . weightedInputsVector [ weightIndex ] ) ; } } } else { var tmpδVector = [ ] ; for ( var outputVectorIndex = 0 ; outputVectorIndex < this . outputVector ! . length ; outputVectorIndex ++ ) { for ( const conn of this . outputs ) { var outputNeuron = conn . out ; for ( var outputIndexOfNextLayer = outputVectorIndex - outputNeuron . timeDelayed + 1 ; outputIndexOfNextLayer <= outputVectorIndex && outputIndexOfNextLayer < outputNeuron . outputVector ! . length ; outputIndexOfNextLayer ++ ) { if ( outputIndexOfNextLayer >= 0 ) { if ( tmpδVector [ outputVectorIndex ] == undefined ) tmpδVector [ outputVectorIndex ] = 0 ; tmpδVector [ outputVectorIndex ] += outputNeuron . errorVector [ outputIndexOfNextLayer ] * conn . weightVector ! [ outputVectorIndex - outputIndexOfNextLayer ] ; } } } this . errorVector [ outputVectorIndex ] = tmpδVector [ outputVectorIndex ] * NonLinearities [ this . activation ] . df ( this . weightedInputsVector [ outputVectorIndex ] ) ; } } } } } export default Net ;