import { clContext as nodenCLContext , OpenCLBuffer } from 'nodencl' ; import { RedioPipe , RedioEnd , isValue , Valve , nil } from 'redioactive' ; import { Frame , Filterer , filterer } from 'beamcoder' ; import { VideoFormat } from '../config' ; import { ClJobs } from '../clJobQueue' ; import ImageProcess from '../process/imageProcess' ; import Transform from '../process/transform' ; export interface AnchorParams { x : number ; y : number ; } export interface FillParams { xOffset : number ; yOffset : number ; xScale : number ; yScale : number ; } export type MixerParams = { anchor : AnchorParams ; rotation : number ; fill : FillParams ; volume : number ; } ; export const MixerDefaults = '{\
		"anchor": { "x": 0, "y": 0 },\
		"rotation": 0,\
		"fill": { "xOffset": 0, "yOffset": 0, "xScale": 1, "yScale": 1 },\
		"volume": 1\
	}' ; export class Mixer { private readonly clContext : nodenCLContext ; private readonly consumerFormat : VideoFormat ; private readonly clJobs : ClJobs ; private transform : ImageProcess | null ; private mixAudio ! : RedioPipe < Frame | RedioEnd > ; private mixVideo ! : RedioPipe < OpenCLBuffer | RedioEnd > ; private audMixFilterer : Filterer | null = null ; private mixParams = JSON . parse ( MixerDefaults ) ; private srcLevels : number [ ] = [ ] ; private paused = true ; private running = true ; private audDone = false ; private vidDone = false ; constructor ( clContext , consumerFormat , clJobs ) { this . clContext = clContext ; this . consumerFormat = consumerFormat ; this . clJobs = clJobs ; this . transform = new ImageProcess ( this . clContext , new Transform ( this . clContext , this . consumerFormat . width , this . consumerFormat . height ) , clJobs ) ; } async init ( sourceID , srcAudio , srcVideo , srcFormat ) { const srcSampleRate = srcFormat . audioSampleRate ; const srcAudChannels = srcFormat . audioChannels ; const dstSampleRate = this . consumerFormat . audioSampleRate ; const dstAudChannels = this . consumerFormat . audioChannels ; const dstAudLayout = `${ dstAudChannels }c` ; let filtSpec = '' ; for ( let c = 0 ; c < srcAudChannels ; ++ c ) { this . srcLevels . push ( 1.0 ) ; const panSpec = `pan=${ dstAudLayout }|c${ c % dstAudChannels }=${ this . srcLevels [ c ] }*c0` ; filtSpec += ( c === 0 ? '' : ';\n' ) + `[in${ c }:a]highpass=mix=0, adelay=delays='', acompressor=threshold=1:mix=0, aformat=sample_fmts=fltp, ${ panSpec }[c${ c }:a]` ; } filtSpec += ';\n' ; for ( let c = 0 ; c < srcAudChannels ; ++ c ) filtSpec += `[c${ c }:a]` ; filtSpec += `amix=inputs=${ srcAudChannels }:duration=shortest:weights=` ; for ( let c = 0 ; c < srcAudChannels ; ++ c ) filtSpec += ( c === 0 ? '' : ' ' ) + ( c < dstAudChannels ? '1' : '0' ) ; filtSpec += `, volume=1.0:eval=frame:precision=float[out0:a]` ; const inParams = [ ] ; for ( let s = 0 ; s < srcAudChannels ; ++ s ) inParams . push ( { name : `in${ s }:a` , timeBase : [ 1 , srcSampleRate ] , sampleRate : srcSampleRate , sampleFormat : 'fltp' , channelLayout : '1c' } ) ; this . audMixFilterer = await filterer ( { filterType : 'audio' , inputParams : inParams , outputParams : [ { name : 'out0:a' , sampleRate : dstSampleRate , sampleFormat : 'fltp' , channelLayout : dstAudLayout } ] , filterSpec : filtSpec } ) ; const audMixFilter = async ( frames ) => { if ( isValue ( frames ) ) { if ( ! this . running ) return nil ; if ( ! this . audMixFilterer ) return nil ; const inSpec = [ ] ; frames . forEach ( ( f , i ) => inSpec . push ( { name : `in${ i }:a` , frames : [ f ] } ) ) ; const ff = await this . audMixFilterer . filter ( inSpec ) ; return ff [ 0 ] && ff [ 0 ] . frames . length > 0 ? ff [ 0 ] . frames : nil ; } else { this . audMixFilterer = null ; this . audDone = true ; if ( this . audDone && this . vidDone ) this . running = false ; return frames ; } } ; await this . transform ?. init ( ) ; const numBytesRGBA = this . consumerFormat . width * this . consumerFormat . height * 4 * 4 ; const mixVidValve = async ( frame ) => { if ( isValue ( frame ) ) { if ( ! this . running ) { frame . release ( ) ; return nil ; } const timestamp = frame . timestamp ; const xfDest = await this . clContext . createBuffer ( numBytesRGBA , 'readwrite' , 'coarse' , { width : this . consumerFormat . width , height : this . consumerFormat . height } , `mixer ${ sourceID } ${ timestamp }` ) ; xfDest . timestamp = timestamp ; await this . transform ?. run ( { input : frame , flipH : false , flipV : false , anchorX : this . mixParams . anchor . x - 0.5 , anchorY : this . mixParams . anchor . y - 0.5 , scaleX : this . mixParams . fill . xScale , scaleY : this . mixParams . fill . yScale , rotate : - this . mixParams . rotation / 360.0 , offsetX : - this . mixParams . fill . xOffset , offsetY : - this . mixParams . fill . yOffset , output : xfDest } , { source : sourceID , timestamp : timestamp } , ( ) => frame . release ( ) ) ; await this . clJobs . runQueue ( { source : sourceID , timestamp : timestamp } ) ; return xfDest ; } else { this . clJobs . clearQueue ( sourceID ) ; this . transform ?. finish ( ) ; this . transform = null ; this . vidDone = true ; if ( this . audDone && this . vidDone ) this . running = false ; return frame ; } } ; this . mixAudio = srcAudio . pause ( ( ) => this . paused && this . running ) . valve ( audMixFilter , { oneToMany : true } ) ; this . mixVideo = srcVideo . pause ( ( frame ) => { if ( ! this . running ) { frame = nil ; return false ; } if ( this . paused && isValue ( frame ) ) ( frame as OpenCLBuffer ) . addRef ( ) ; return this . paused ; } ) . valve ( mixVidValve ) ; } setPaused ( pause ) { this . paused = pause ; this . setVolume ( this . mixParams . volume , this . paused ) ; } release ( ) { this . running = false ; } setMixParams ( mixParams ) { this . mixParams = mixParams ; this . setVolume ( this . mixParams . volume ) ; } setVolume ( volume , mute ? ) { this . mixParams . volume = volume ; const volFilter = this . audMixFilterer ?. graph . filters . find ( ( f ) => f . filter . name === 'volume' ) ; if ( volFilter && volFilter . priv ) volFilter . priv = { volume : mute ? '0.0' : this . mixParams . volume . toString ( ) } ; return true ; } getAudioPipe ( ) { return this . mixAudio ; } getVideoPipe ( ) { return this . mixVideo ; } }