import { CanvasVideoFrameBuffer , VideoFrameBuffer , VideoFrameProcessor } from "amazon-chime-sdk-js" ; import { VirtualBackgroundGoogleMeetTFLite } from "./VirtualBackgroundGoogleMeetTFLite" ; import { VirtualBackgroundGoogleMeetTFJS } from "./VirtualBackgroundGoogleMeetTFJS" ; import { VirtualBackgroundBodyPix } from "./VirtualBackgroundBodyPix" ; export type VirtualBackgroundSegmentationType = "None" | "BodyPix" | "GoogleMeet" | "GoogleMeetTFLite" ; export type BackgroundMode = "Image" | "Blur" | "Color" ; export interface VirtualBackgroundConfig { frontPositionX : number ; frontPositionY : number ; frontWidth : number ; frontHeight : number ; width : number ; height : number ; type : VirtualBackgroundSegmentationType ; backgroundMode : BackgroundMode ; backgroundImage : HTMLCanvasElement | HTMLImageElement | null ; backgroundColor : string ; } export class VirtualBackground implements VideoFrameProcessor { BodyPix = new VirtualBackgroundBodyPix ( ) ; GMTfjs = new VirtualBackgroundGoogleMeetTFJS ( ) ; GMTflite = new VirtualBackgroundGoogleMeetTFLite ( ) ; canvasBackground = document . createElement ( "canvas" ) ; private config : VirtualBackgroundConfig = { frontPositionX : 0 , frontPositionY : 0 , frontWidth : 1 , frontHeight : 1 , width : - 1 , height : - 1 , type : "None" , backgroundMode : "Color" , backgroundImage : null , backgroundColor : "#000000" , } ; targetCanvas = document . createElement ( "canvas" ) ; canvasVideoFrameBuffer = new CanvasVideoFrameBuffer ( this . targetCanvas ! ) ; constructor ( ) { console . log ( `[VirtualBackground][constructor] initializing.` ) ; const bg = new Image ( ) ; bg . src = "/bg1.jpg" ; bg . onload = ( ) => { this . canvasBackground . getContext ( "2d" ) ! . drawImage ( bg , 0 , 0 , this . canvasBackground . width , this . canvasBackground . height ) ; } ; } async destroy ( ) { this . canvasVideoFrameBuffer . destroy ( ) ; return ; } setVirtualBackgroundType ( type ) { this . config . type = type ; } setBackgroundMode ( mode ) { this . config . backgroundMode = mode ; } setBackgroundImage ( path ) { const bg = new Image ( ) ; bg . src = path ; bg . onload = ( ) => { this . canvasBackground . getContext ( "2d" ) ! . drawImage ( bg , 0 , 0 , this . canvasBackground . width , this . canvasBackground . height ) ; } ; } setBackgroundColor ( color ) { this . config . backgroundColor = color ; } async process ( buffers ) { if ( buffers . length === 0 ) { return Promise . resolve ( buffers ) ; } const canvas = buffers [ 0 ] . asCanvasElement ( ) ; const frameWidth = canvas ! . width ; const frameHeight = canvas ! . height ; if ( frameWidth === 0 || frameHeight === 0 ) { return Promise . resolve ( buffers ) ; } for ( const f of buffers ) { try { const canvas = f . asCanvasElement ( ) as HTMLCanvasElement ; switch ( this . config . type ) { case "BodyPix" : this . BodyPix . predict ( canvas , this . canvasBackground , this . targetCanvas , this . config ) ; break ; case "GoogleMeet" : this . GMTfjs . predict ( canvas , this . canvasBackground , this . targetCanvas , this . config ) ; break ; case "GoogleMeetTFLite" : this . GMTflite . predict ( canvas , this . canvasBackground , this . targetCanvas , this . config ) ; break ; default : this . convert_none ( canvas ) ; } } catch ( err ) { console . log ( "Exception:: " , err ) ; } } buffers [ 0 ] = this . canvasVideoFrameBuffer ; return Promise . resolve ( buffers ) ; } convert_none = ( foreground ) => { this . targetCanvas . getContext ( "2d" ) ! . drawImage ( foreground , 0 , 0 , this . targetCanvas . width , this . targetCanvas . height ) ; } ; }