import { HandPoseConfig , HandPoseOperatipnParams , HandPoseFunctionType , WorkerCommand , WorkerResponse , } from "./const" ; import { getBrowserType , BrowserType } from "./BrowserUtil" ; import * as handpose from "@tensorflow-models/handpose" ; import * as tf from "@tensorflow/tfjs" ; import { setWasmPath } from "@tensorflow/tfjs-backend-wasm" ; export { HandPoseConfig , HandPoseOperatipnParams } ; import workerJs from "worker-loader?inline=no-fallback!./handpose-worker-worker.ts" ; export const generateHandPoseDefaultConfig = ( ) : HandPoseConfig => { const defaultConf = { browserType : getBrowserType ( ) , model : { maxContinuousChecks : Infinity , detectionConfidence : 0.8 , iouThreshold : 0.3 , scoreThreshold : 0.75 , } , useTFWasmBackend : false , processOnLocal : false , modelReloadInterval : 1024 * 60 , wasmPath : "/tfjs-backend-wasm.wasm" , pageUrl : window . location . href , } ; if ( defaultConf . browserType == BrowserType . SAFARI ) { defaultConf . processOnLocal = true ; } return defaultConf ; } ; export const generateDefaultHandPoseParams = ( ) => { const defaultParams = { type : HandPoseFunctionType . EstimateHands , estimateHands : { flipHorizontal : false , } , processWidth : 300 , processHeight : 300 , } ; return defaultParams ; } ; export class LocalHP { model : handpose . HandPose | null = null ; canvas : HTMLCanvasElement = ( ( ) => { const newCanvas = document . createElement ( "canvas" ) ; newCanvas . style . display = "none" ; return newCanvas ; } ) ( ) ; init = ( config ) => { return handpose . load ( config . model ) . then ( ( res ) => { console . log ( "handpose loaded locally" , config ) ; this . model = res ; return ; } ) ; } ; predict = async ( targetCanvas , config , params ) : Promise < handpose . AnnotatedPrediction [ ] > => { console . log ( "current backend[main thread]:" , tf . getBackend ( ) ) ; const processWidth = params . processWidth <= 0 || params . processHeight <= 0 ? targetCanvas . width : params . processWidth ; const processHeight = params . processWidth <= 0 || params . processHeight <= 0 ? targetCanvas . height : params . processHeight ; this . canvas . width = processWidth ; this . canvas . height = processHeight ; const ctx = this . canvas . getContext ( "2d" ) ! ; ctx . drawImage ( targetCanvas , 0 , 0 , processWidth , processHeight ) ; const newImg = ctx . getImageData ( 0 , 0 , processWidth , processHeight ) ; const prediction = await this . model ! . estimateHands ( newImg ) ; return prediction ; } ; } export class HandPoseWorkerManager { private workerHP : Worker | null = null ; private canvas = document . createElement ( "canvas" ) ; private config = generateHandPoseDefaultConfig ( ) ; private localHP = new LocalHP ( ) ; private initializeModel_internal = ( ) => { const workerHP = new workerJs ( ) ; workerHP ! . postMessage ( { message : WorkerCommand . INITIALIZE , config : this . config , } ) ; const p = new Promise < void > ( ( onResolve , onFail ) => { workerHP ! . onmessage = ( event ) => { if ( event . data . message === WorkerResponse . INITIALIZED ) { console . log ( "WORKERSS INITIALIZED" ) ; this . workerHP = workerHP ; onResolve ( ) ; } else { console . log ( "Handpose Initialization something wrong.." ) ; onFail ( event ) ; } } ; } ) ; return p ; } ; init = async ( config ) => { if ( config != null ) { this . config = config ; } if ( this . workerHP ) { this . workerHP . terminate ( ) ; } this . workerHP = null ; if ( this . config . processOnLocal === true ) { const dirname = this . config . pageUrl . substr ( 0 , this . config . pageUrl . lastIndexOf ( "/" ) ) ; const wasmPath = `${ dirname }${ this . config . wasmPath }` ; console . log ( `use wasm backend ${ wasmPath }` ) ; setWasmPath ( wasmPath ) ; if ( this . config . useTFWasmBackend ) { require ( "@tensorflow/tfjs-backend-wasm" ) ; await tf . setBackend ( "wasm" ) ; } else { console . log ( "use webgl backend" ) ; require ( "@tensorflow/tfjs-backend-webgl" ) ; await tf . setBackend ( "webgl" ) ; } await tf . ready ( ) ; await this . localHP ! . init ( this . config ! ) ; return ; } return this . initializeModel_internal ( ) ; } ; model_reload_interval = 0 ; model_refresh_counter = 0 ; private liveCheck = ( ) => { if ( this . model_refresh_counter > this . config . modelReloadInterval && this . config . modelReloadInterval > 0 ) { console . log ( "reload model! this is a work around for memory leak." ) ; this . model_refresh_counter = 0 ; if ( this . config . browserType === BrowserType . SAFARI && this . config . processOnLocal === true ) { return new Promise < void > ( ( onResolve , onFail ) => { tf . ready ( ) . then ( ( ) => { this . localHP . init ( this . config ! ) . then ( ( ) => { onResolve ( ) ; } ) ; } ) ; } ) ; } else { this . workerHP ?. terminate ( ) ; return this . initializeModel_internal ( ) ; } } else { this . model_refresh_counter += 1 ; } } ; predict = async ( targetCanvas , params ) => { if ( this . config . processOnLocal === true ) { const prediction = await this . localHP . predict ( targetCanvas , this . config , params ) ; return prediction ; } if ( ! this . workerHP ) { return null ; } if ( this . config . browserType == BrowserType . SAFARI ) { let data ; if ( params . processHeight > 0 && params . processWidth > 0 ) { this . canvas . width = params . processWidth ; this . canvas . height = params . processHeight ; this . canvas . getContext ( "2d" ) ! . drawImage ( targetCanvas , 0 , 0 , params . processWidth , params . processHeight ) ; data = this . canvas . getContext ( "2d" ) ! . getImageData ( 0 , 0 , params . processWidth , params . processHeight ) . data ; } else { const ctx = targetCanvas . getContext ( "2d" ) ! ; data = ctx . getImageData ( 0 , 0 , targetCanvas . width , targetCanvas . height ) ! . data ; } const uid = performance . now ( ) ; const p = new Promise ( async ( onResolve , onFail ) => { await this . liveCheck ( ) ; this . workerHP ! . postMessage ( { message : WorkerCommand . PREDICT , uid : uid , config : this . config , params : params , data : data , } , [ data . buffer ] ) ; this . workerHP ! . onmessage = ( event ) => { if ( event . data . message === WorkerResponse . PREDICTED && event . data . uid === uid ) { onResolve ( event . data . prediction ) ; } else { console . log ( "Handpose Prediction something wrong.." ) ; } } ; } ) ; return p ; } else { const offscreen = new OffscreenCanvas ( targetCanvas . width , targetCanvas . height ) ; const offctx = offscreen . getContext ( "2d" ) ! ; offctx . drawImage ( targetCanvas , 0 , 0 , targetCanvas . width , targetCanvas . height ) ; const imageBitmap = offscreen . transferToImageBitmap ( ) ; const uid = performance . now ( ) ; const p = new Promise ( async ( onResolve , onFail ) => { await this . liveCheck ( ) ; this . workerHP ! . postMessage ( { message : WorkerCommand . PREDICT , uid : uid , config : this . config , params : params , image : imageBitmap , } , [ imageBitmap ] ) ; this . workerHP ! . onmessage = ( event ) => { if ( event . data . message === WorkerResponse . PREDICTED && event . data . uid === uid ) { onResolve ( event . data . prediction ) ; } else { console . log ( "Handpose Prediction something wrong.." ) ; } } ; } ) ; return p ; } } ; } const fingerLookupIndices = { thumb : [ 0 , 1 , 2 , 3 , 4 ] , indexFinger : [ 0 , 5 , 6 , 7 , 8 ] , middleFinger : [ 0 , 9 , 10 , 11 , 12 ] , ringFinger : [ 0 , 13 , 14 , 15 , 16 ] , pinky : [ 0 , 17 , 18 , 19 , 20 ] , } ; export const drawHandSkelton = ( srcCanvas , prediction , params ) => { const canvas = document . createElement ( "canvas" ) ; canvas . width = srcCanvas . width ; canvas . height = srcCanvas . height ; const ctx = canvas . getContext ( "2d" ) ! ; const scaleX = srcCanvas . width / params . processWidth ; const scaleY = srcCanvas . height / params . processHeight ; prediction . forEach ( ( x ) => { const landmarks = x . landmarks as number [ ] [ ] ; landmarks . forEach ( ( landmark ) => { const x = landmark [ 0 ] * scaleX ; const y = landmark [ 1 ] * scaleY ; ctx . fillRect ( x , y , 5 , 5 ) ; } ) ; const fingers = Object . keys ( fingerLookupIndices ) ; fingers . forEach ( ( x ) => { const points = fingerLookupIndices [ x ] . map ( ( idx ) => landmarks [ idx ] ) ; ctx . beginPath ( ) ; ctx . moveTo ( points [ 0 ] [ 0 ] * scaleX , points [ 0 ] [ 1 ] * scaleY ) ; for ( let i = 1 ; i < points . length ; i ++ ) { const point = points [ i ] ; ctx . lineTo ( point [ 0 ] * scaleX , point [ 1 ] * scaleY ) ; } ctx . lineWidth = 3 ; ctx . stroke ( ) ; ctx . closePath ( ) ; } ) ; } ) ; const image = ctx . getImageData ( 0 , 0 , canvas . width , canvas . height ) ; canvas . remove ( ) ; return image ; } ;