import * as tf from '@tensorflow/tfjs-core' ; import { Worker } from './controller.worker' ; import { Tracker } from './image-target/tracker/tracker' ; import { CropDetector } from './image-target/detector/crop-detector' ; import { InputLoader } from './image-target/input-loader' ; import { Compiler } from './compiler' ; import { fetchFunc } from 'fetch-wechat' ; const INTERPOLATION_FACTOR = 5 ; const WARMUP_COUNT_TOLERANCE = 10 ; const MISS_COUNT_TOLERANCE = 30 ; export class Controller { tracker : Tracker | undefined ; inputWidth : any ; inputHeight : any ; cropDetector : CropDetector ; inputLoader : InputLoader ; markerDimensions : any ; onUpdate : any ; debugMode : boolean ; processingVideo : boolean ; interestedTargetIndex : any ; maxTrack : number ; projectionTransform : number [ ] [ ] ; projectionMatrix : number [ ] ; worker : Worker ; workerMatchDone : any ; workerTrackDone : any ; shouldCaptureRegion : boolean ; capturedRegion : null ; trackingStates : any [ ] ; constructor ( { inputWidth , inputHeight , onUpdate = null , maxTrack = 1 , debugMode = false } ) { this . inputWidth = inputWidth ; this . inputHeight = inputHeight ; this . maxTrack = maxTrack ; this . cropDetector = new CropDetector ( this . inputWidth , this . inputHeight , debugMode ) ; this . inputLoader = new InputLoader ( ) ; this . markerDimensions = null ; this . onUpdate = onUpdate ; this . debugMode = debugMode ; this . processingVideo = false ; this . shouldCaptureRegion = false ; this . capturedRegion = null ; this . interestedTargetIndex = - 1 ; this . trackingStates = [ ] ; const near = 10 ; const far = 10000 ; const fovy = 45.0 * Math . PI / 180 ; const f = ( this . inputHeight / 2 ) / Math . tan ( fovy / 2 ) ; this . projectionTransform = [ [ f , 0 , this . inputWidth / 2 ] , [ 0 , f , this . inputHeight / 2 ] , [ 0 , 0 , 1 ] ] ; this . projectionMatrix = this . _glProjectionMatrix ( { projectionTransform : this . projectionTransform , width : this . inputWidth , height : this . inputHeight , near : near , far : far , } ) ; this . worker = new Worker ( ) ; this . workerMatchDone = null ; this . workerTrackDone = null ; this . worker . onmessage = ( e ) => { if ( e . data . type === 'matchDone' && this . workerMatchDone !== null ) { this . workerMatchDone ( e . data ) ; } if ( e . data . type === 'trackUpdateDone' && this . workerTrackDone !== null ) { this . workerTrackDone ( e . data ) ; } } ; } showTFStats ( ) { console . log ( tf . memory ( ) . numTensors ) ; console . table ( tf . memory ( ) ) ; } async addImageTargets ( fileURL ) { const content = await fetchFunc ( ) ( fileURL , { } ) ; const buffer = await content . arrayBuffer ( ) ; const result = this . addImageTargetsFromBuffer ( buffer ) ; return result ; } addImageTargetsFromBuffer ( buffer ) { const compiler = new Compiler ( ) ; const dataList = compiler . importData ( buffer ) ; console . log ( 'dataList' , dataList ) ; const trackingDataList = [ ] ; const matchingDataList = [ ] ; const dimensions = [ ] ; for ( let i = 0 ; i < dataList . length ; i ++ ) { matchingDataList . push ( dataList [ i ] . matchingData ) ; trackingDataList . push ( dataList [ i ] . trackingData ) ; dimensions . push ( [ dataList [ i ] . targetImage . width , dataList [ i ] . targetImage . height ] ) ; } this . tracker = new Tracker ( dimensions , trackingDataList , this . projectionTransform , this . inputWidth , this . inputHeight , this . debugMode ) ; this . worker . postMessage ( { type : 'setup' , inputWidth : this . inputWidth , inputHeight : this . inputHeight , projectionTransform : this . projectionTransform , debugMode : this . debugMode , matchingDataList , } ) ; this . markerDimensions = dimensions ; this . trackingStates = [ ] ; for ( let i = 0 ; i < this . markerDimensions . length ; i ++ ) { this . trackingStates . push ( { showing : false , isTracking : false , currentModelViewTransform : null , trackCount : 0 , trackMiss : 0 } ) ; } return { dimensions : dimensions , matchingDataList , trackingDataList } ; } dummyRun ( input ) { const inputT = this . inputLoader . loadInput ( input ) ; this . cropDetector . detect ( inputT ) ; this . tracker . dummyRun ( inputT ) ; inputT . dispose ( ) ; } getProjectionMatrix ( ) { return this . projectionMatrix ; } getWorldMatrix ( modelViewTransform , targetIndex ) { return this . _glModelViewMatrix ( modelViewTransform , targetIndex ) ; } async _detectAndMatch ( inputT , targetIndexes ) { const { featurePoints } = this . cropDetector . detectMoving ( inputT ) ; const { targetIndex : matchedTargetIndex , modelViewTransform } = await this . _workerMatch ( featurePoints , targetIndexes ) ; return { targetIndex : matchedTargetIndex , modelViewTransform } ; } async _trackAndUpdate ( inputT , lastModelViewTransform , targetIndex ) { const startTime = new Date ( ) . getTime ( ) ; const { worldCoords , screenCoords } = this . tracker . track ( inputT , lastModelViewTransform , targetIndex ) ; console . log ( 'tracker track' , new Date ( ) . getTime ( ) - startTime ) ; if ( worldCoords . length < 4 ) return null ; const modelViewTransform = await this . _workerTrackUpdate ( lastModelViewTransform , { worldCoords , screenCoords } ) ; console . log ( '_workerTrackUpdate' , new Date ( ) . getTime ( ) - startTime ) ; return modelViewTransform ; } async processVideo ( input ) { if ( this . processingVideo ) return ; this . processingVideo = true ; const startProcessing = async ( ) => { { if ( ! this . processingVideo ) return ; const inputT = this . inputLoader . loadInput ( input ) ; let shouldCapture = false ; const nTracking = this . trackingStates . reduce ( ( acc , s ) => { return acc + ( ! ! s . isTracking ? 1 : 0 ) ; } , 0 ) ; if ( nTracking < this . maxTrack ) { const matchingIndexes = [ ] ; for ( let i = 0 ; i < this . trackingStates . length ; i ++ ) { const trackingState = this . trackingStates [ i ] ; if ( trackingState . isTracking === true ) continue ; if ( this . interestedTargetIndex !== - 1 && this . interestedTargetIndex !== i ) continue ; matchingIndexes . push ( i ) ; } const { targetIndex : matchedTargetIndex , modelViewTransform } = await this . _detectAndMatch ( inputT , matchingIndexes ) ; if ( matchedTargetIndex !== - 1 ) { this . trackingStates [ matchedTargetIndex ] . isTracking = true ; this . trackingStates [ matchedTargetIndex ] . currentModelViewTransform = modelViewTransform ; if ( this . shouldCaptureRegion ) { shouldCapture = true ; } } } for ( let i = 0 ; i < this . trackingStates . length ; i ++ ) { const trackingState = this . trackingStates [ i ] ; if ( trackingState . isTracking ) { let modelViewTransform = await this . _trackAndUpdate ( inputT , trackingState . currentModelViewTransform , i ) ; if ( modelViewTransform === null ) { trackingState . isTracking = false ; } else { trackingState . currentModelViewTransform = modelViewTransform ; } } if ( ! trackingState . showing ) { if ( trackingState . isTracking ) { trackingState . trackMiss = 0 ; trackingState . trackCount += 1 ; if ( trackingState . trackCount > WARMUP_COUNT_TOLERANCE ) { trackingState . showing = true ; trackingState . trackingMatrix = null ; } } } if ( trackingState . showing ) { if ( ! trackingState . isTracking ) { trackingState . trackCount = 0 ; trackingState . trackMiss += 1 ; if ( trackingState . trackMiss > MISS_COUNT_TOLERANCE ) { trackingState . showing = false ; trackingState . trackingMatrix = null ; this . onUpdate && this . onUpdate ( { type : 'updateMatrix' , targetIndex : i , worldMatrix : null } ) ; } } else { trackingState . trackMiss = 0 ; } } if ( trackingState . showing ) { const worldMatrix = this . _glModelViewMatrix ( trackingState . currentModelViewTransform , i ) ; if ( trackingState . trackingMatrix === null ) { trackingState . trackingMatrix = worldMatrix ; } else { for ( let j = 0 ; j < worldMatrix . length ; j ++ ) { trackingState . trackingMatrix [ j ] = trackingState . trackingMatrix [ j ] + ( worldMatrix [ j ] - trackingState . trackingMatrix [ j ] ) / INTERPOLATION_FACTOR ; } } const clone = [ ] ; for ( let j = 0 ; j < worldMatrix . length ; j ++ ) { clone [ j ] = trackingState . trackingMatrix [ j ] ; } this . onUpdate && this . onUpdate ( { type : 'updateMatrix' , targetIndex : i , worldMatrix : clone } ) ; } } inputT . dispose ( ) ; this . onUpdate && this . onUpdate ( { type : 'processDone' } ) ; await tf . nextFrame ( ) ; this . processingVideo = false ; } } ; await startProcessing ( ) ; } stopProcessVideo ( ) { this . processingVideo = false ; } async detect ( input ) { const inputT = this . inputLoader . loadInput ( input ) ; const featurePoints = await this . cropDetector . detect ( inputT ) ; inputT . dispose ( ) ; return featurePoints ; } async match ( featurePoints , targetIndex ) { const { modelViewTransform , debugExtra } = await this . _workerMatch ( featurePoints , [ targetIndex ] ) ; return { modelViewTransform , debugExtra } ; } async track ( input , modelViewTransforms , targetIndex ) { const inputT = this . inputLoader . loadInput ( input ) ; const result = this . tracker . track ( inputT , modelViewTransforms , targetIndex ) ; inputT . dispose ( ) ; return result ; } async trackAllFrames ( input , modelViewTransforms , targetIndex , nKeyframes ) { const inputT = this . inputLoader . loadInput ( input ) ; const trackResults = [ ] ; for ( let i = 0 ; i < nKeyframes ; i ++ ) { const result = this . tracker . track ( inputT , modelViewTransforms , targetIndex , i ) ; trackResults . push ( result ) ; } inputT . dispose ( ) ; return trackResults ; } async trackUpdate ( modelViewTransform , trackFeatures ) { if ( trackFeatures . worldCoords . length < 4 ) return null ; const modelViewTransform2 = await this . _workerTrackUpdate ( modelViewTransform , trackFeatures ) ; return modelViewTransform2 ; } _workerMatch ( featurePoints , targetIndexes ) { return new Promise ( async ( resolve , _ ) => { this . workerMatchDone = ( data ) => { resolve ( { targetIndex : data . targetIndex , modelViewTransform : data . modelViewTransform , debugExtra : data . debugExtra } ) ; } ; this . worker . postMessage ( { type : 'match' , featurePoints : featurePoints , targetIndexes } ) ; } ) ; } _workerTrackUpdate ( modelViewTransform , trackingFeatures ) { return new Promise ( async ( resolve ) => { this . workerTrackDone = ( data ) => { resolve ( data . modelViewTransform ) ; } ; const { worldCoords , screenCoords } = trackingFeatures ; this . worker . postMessage ( { type : 'trackUpdate' , modelViewTransform , worldCoords , screenCoords } ) ; } ) ; } _glModelViewMatrix ( modelViewTransform , targetIndex ) { const height = this . markerDimensions [ targetIndex ] [ 1 ] ; const openGLWorldMatrix = [ modelViewTransform [ 0 ] [ 0 ] , - modelViewTransform [ 1 ] [ 0 ] , - modelViewTransform [ 2 ] [ 0 ] , 0 , - modelViewTransform [ 0 ] [ 1 ] , modelViewTransform [ 1 ] [ 1 ] , modelViewTransform [ 2 ] [ 1 ] , 0 , - modelViewTransform [ 0 ] [ 2 ] , modelViewTransform [ 1 ] [ 2 ] , modelViewTransform [ 2 ] [ 2 ] , 0 , modelViewTransform [ 0 ] [ 1 ] * height + modelViewTransform [ 0 ] [ 3 ] , - ( modelViewTransform [ 1 ] [ 1 ] * height + modelViewTransform [ 1 ] [ 3 ] ) , - ( modelViewTransform [ 2 ] [ 1 ] * height + modelViewTransform [ 2 ] [ 3 ] ) , 1 ] ; return openGLWorldMatrix ; } _glProjectionMatrix ( { projectionTransform , width , height , near , far } ) { const proj = [ [ 2 * projectionTransform [ 0 ] [ 0 ] / width , 0 , - ( 2 * projectionTransform [ 0 ] [ 2 ] / width - 1 ) , 0 ] , [ 0 , 2 * projectionTransform [ 1 ] [ 1 ] / height , - ( 2 * projectionTransform [ 1 ] [ 2 ] / height - 1 ) , 0 ] , [ 0 , 0 , - ( far + near ) / ( far - near ) , - 2 * far * near / ( far - near ) ] , [ 0 , 0 , - 1 , 0 ] ] ; const projMatrix = [ ] ; for ( let i = 0 ; i < 4 ; i ++ ) { for ( let j = 0 ; j < 4 ; j ++ ) { projMatrix . push ( proj [ j ] [ i ] ) ; } } return projMatrix ; } }