import { Sink } from '../component' ; import { Clock } from '../../utils/clock' ; import { Scheduler } from '../../utils/scheduler' ; import { Message , MessageType } from '../message' ; import { Writable , Readable } from 'stream' ; import { VideoMedia } from '../../utils/protocols/sdp' ; interface BlobMessage { readonly blob : Blob ; readonly ntpTimestamp : number ; } type BlobMessageHandler = ( msg : BlobMessage ) => void ; interface RateInfo { bitrate : number ; framerate : number ; renderedFrames : number ; } const resetInfo = ( info ) => { info . bitrate = 0 ; info . framerate = 0 ; info . renderedFrames = 0 ; } ; interface ByteDuration { byteLength : number ; duration : number ; } const generateUpdateInfo = ( clockrate ) => { let cumulativeByteLength = 0 ; let cumulativeDuration = 0 ; let cumulativeFrames = 0 ; return ( info , { byteLength , duration } ) => { cumulativeByteLength += byteLength ; cumulativeDuration += duration ; cumulativeFrames ++ ; if ( cumulativeDuration >= clockrate ) { const bits = 8 * cumulativeByteLength ; const frames = cumulativeFrames ; const seconds = cumulativeDuration / clockrate ; info . bitrate = bits / seconds ; info . framerate = frames / seconds ; cumulativeByteLength = 0 ; cumulativeDuration = 0 ; cumulativeFrames = 0 ; } } ; } ; /**
 * Canvas component
 *
 * Draws an incoming stream of JPEG images onto a <canvas> element.
 * The RTP timestamps are used to schedule the drawing of the images.
 * An instance can be used as a 'clock' itself, e.g. with a scheduler.
 *
 * The following handlers can be set on a component instance:
 *  - onCanplay: will be called when the first frame is ready and
 *               the correct frame size has been set on the canvas.
 *               At this point, the clock can be started by calling
 *               `.play()` method on the component.
 *  - onSync: will be called when the presentation time offset is
 *            known, with the latter as argument (in UNIX milliseconds)
 *
 */ export class CanvasSink extends Sink { public onCanplay ? : ( ) => void ; public onSync ? : ( ntpPresentationTime : number ) => void ; private readonly _clock : Clock ; private readonly _scheduler : Scheduler < BlobMessage > ; private readonly _info : RateInfo ; param el constructor ( el ) { if ( el === undefined ) { throw new Error ( 'canvas element argument missing' ) ; } let firstTimestamp = 0 ; let lastTimestamp = 0 ; let clockrate = 0 ; const info = { bitrate : 0 , framerate : 0 , renderedFrames : 0 , } ; let updateInfo ; let ctx = null ; if ( window . createImageBitmap !== undefined ) { ctx = el . getContext ( 'bitmaprenderer' ) ; } if ( ctx === null ) { ctx = el . getContext ( '2d' ) ; } let drawImageBlob ; if ( ctx === null ) { drawImageBlob = ( ) => { } ; } else if ( 'transferFromImageBitmap' in ctx ) { const ctxBitmaprenderer = ctx ; drawImageBlob = ( { blob } ) => { info . renderedFrames ++ ; window . createImageBitmap ( blob ) . then ( ( imageBitmap ) => { ctxBitmaprenderer . transferFromImageBitmap ( imageBitmap ) ; } ) . catch ( ( ) => { } ) ; } ; } else { const ctx2d = ctx ; const img = new Image ( ) ; img . onload = ( ) => { ctx2d . drawImage ( img , 0 , 0 ) ; } ; drawImageBlob = ( { blob } ) => { info . renderedFrames ++ ; const url = window . URL . createObjectURL ( blob ) ; img . src = url ; } ; } const clock = new Clock ( ) ; const scheduler = new Scheduler ( clock , drawImageBlob ) ; let ntpPresentationTime = 0 ; const onCanplay = ( ) => { this . onCanplay && this . onCanplay ( ) ; } ; const onSync = ( ntpPresentationTime ) => { this . onSync && this . onSync ( ntpPresentationTime ) ; } ; const incoming = new Writable ( { objectMode : true , write : ( msg , encoding , callback ) => { if ( msg . type === MessageType . SDP ) { clock . reset ( ) ; scheduler . reset ( ) ; firstTimestamp = 0 ; const jpegMedia = msg . sdp . media . find ( ( media ) : media is VideoMedia => { return ( media . type === 'video' && media . rtpmap !== undefined && media . rtpmap . encodingName === 'JPEG' ) ; } ) ; if ( jpegMedia !== undefined && jpegMedia . rtpmap !== undefined ) { clockrate = jpegMedia . rtpmap . clockrate ; resetInfo ( info ) ; updateInfo = generateUpdateInfo ( clockrate ) ; } callback ( ) ; } else if ( msg . type === MessageType . JPEG ) { const { timestamp , ntpTimestamp } = msg ; if ( ! firstTimestamp ) { firstTimestamp = timestamp ; lastTimestamp = timestamp ; const { width , height } = msg . framesize ; el . width = width ; el . height = height ; scheduler . init ( 0 ) ; } const presentationTime = ( 1000 * ( timestamp - firstTimestamp ) ) / clockrate ; const blob = new window . Blob ( [ msg . data ] , { type : 'image/jpeg' } ) ; if ( ! ntpPresentationTime && ntpTimestamp ) { ntpPresentationTime = ntpTimestamp - presentationTime ; onSync ( ntpPresentationTime ) ; } scheduler . run ( { ntpTimestamp : presentationTime , blob , } ) ; if ( timestamp === firstTimestamp ) { onCanplay ( ) ; } updateInfo ( info , { byteLength : msg . data . length , duration : timestamp - lastTimestamp , } ) ; lastTimestamp = timestamp ; callback ( ) ; } else { callback ( ) ; } } , } ) ; const outgoing = new Readable ( { objectMode : true , read : function ( ) { } , } ) ; outgoing . on ( 'error' , ( ) => { console . warn ( 'outgoing stream broke somewhere' ) ; } ) ; super ( incoming , outgoing ) ; this . _clock = clock ; this . _scheduler = scheduler ; this . _info = info ; this . onCanplay = undefined ; this . onSync = undefined ; } /**
     * Retrieve the current presentation time (seconds)
     */ get currentTime ( ) { return this . _clock . currentTime ; } /**
     * Pause the presentation.
     */ pause ( ) { this . _scheduler . suspend ( ) ; this . _clock . pause ( ) ; } /**
     * Start the presentation.
     */ play ( ) { this . _clock . play ( ) ; this . _scheduler . resume ( ) ; } get bitrate ( ) { return this . _info . bitrate ; } get framerate ( ) { return this . _info . framerate ; } }