import { ParameterType } from "../../modules/plugins" ; import { unique } from "../utils" ; const preloadParameterTypes = < const > [ ParameterType . AUDIO , ParameterType . IMAGE , ParameterType . VIDEO , ] ; type PreloadType = typeof preloadParameterTypes [ number ] ; export class MediaAPI { constructor ( private useWebaudio , private webaudioContext ? ) { } private video_buffers = { } ; getVideoBuffer ( videoID ) { return this . video_buffers [ videoID ] ; } private context = null ; private audio_buffers = [ ] ; initAudio ( ) { this . context = this . useWebaudio ? this . webaudioContext : null ; } audioContext ( ) { if ( this . context !== null ) { if ( this . context . state !== "running" ) { this . context . resume ( ) ; } } return this . context ; } getAudioBuffer ( audioID ) { return new Promise ( ( resolve , reject ) => { if ( typeof this . audio_buffers [ audioID ] == "undefined" || this . audio_buffers [ audioID ] == "tmp" ) { this . preloadAudio ( [ audioID ] , ( ) => { resolve ( this . audio_buffers [ audioID ] ) ; } , ( ) => { } , ( e ) => { reject ( e . error ) ; } ) ; } else { resolve ( this . audio_buffers [ audioID ] ) ; } } ) ; } private preload_requests = [ ] ; private img_cache = { } ; preloadAudio ( files , callback_complete = ( ) => { } , callback_load = ( filepath ) => { } , callback_error = ( error_msg ) => { } ) { files = unique ( files . flat ( ) ) ; let n_loaded = 0 ; if ( files . length == 0 ) { callback_complete ( ) ; return ; } const load_audio_file_webaudio = ( source , count = 1 ) => { const request = new XMLHttpRequest ( ) ; request . open ( "GET" , source , true ) ; request . responseType = "arraybuffer" ; request . onload = ( ) => { this . context . decodeAudioData ( request . response , ( buffer ) => { this . audio_buffers [ source ] = buffer ; n_loaded ++ ; callback_load ( source ) ; if ( n_loaded == files . length ) { callback_complete ( ) ; } } , ( e ) => { callback_error ( { source : source , error : e } ) ; } ) ; } ; request . onerror = function ( e ) { let err = e ; if ( this . status == 404 ) { err = "404" ; } callback_error ( { source : source , error : err } ) ; } ; request . onloadend = function ( e ) { if ( this . status == 404 ) { callback_error ( { source : source , error : "404" } ) ; } } ; request . send ( ) ; this . preload_requests . push ( request ) ; } ; const load_audio_file_html5audio = ( source , count = 1 ) => { const audio = new Audio ( ) ; const handleCanPlayThrough = ( ) => { this . audio_buffers [ source ] = audio ; n_loaded ++ ; callback_load ( source ) ; if ( n_loaded == files . length ) { callback_complete ( ) ; } audio . removeEventListener ( "canplaythrough" , handleCanPlayThrough ) ; } ; audio . addEventListener ( "canplaythrough" , handleCanPlayThrough ) ; audio . addEventListener ( "error" , function handleError ( e ) { callback_error ( { source : audio . src , error : e } ) ; audio . removeEventListener ( "error" , handleError ) ; } ) ; audio . addEventListener ( "abort" , function handleAbort ( e ) { callback_error ( { source : audio . src , error : e } ) ; audio . removeEventListener ( "abort" , handleAbort ) ; } ) ; audio . src = source ; this . preload_requests . push ( audio ) ; } ; for ( const file of files ) { if ( typeof this . audio_buffers [ file ] !== "undefined" ) { n_loaded ++ ; callback_load ( file ) ; if ( n_loaded == files . length ) { callback_complete ( ) ; } } else { this . audio_buffers [ file ] = "tmp" ; if ( this . audioContext ( ) !== null ) { load_audio_file_webaudio ( file ) ; } else { load_audio_file_html5audio ( file ) ; } } } } preloadImages ( images , callback_complete = ( ) => { } , callback_load = ( filepath ) => { } , callback_error = ( error_msg ) => { } ) { images = unique ( images . flat ( ) ) ; var n_loaded = 0 ; if ( images . length === 0 ) { callback_complete ( ) ; return ; } for ( var i = 0 ; i < images . length ; i ++ ) { var img = new Image ( ) ; img . onload = function ( ) { n_loaded ++ ; callback_load ( img . src ) ; if ( n_loaded === images . length ) { callback_complete ( ) ; } } ; img . onerror = function ( e ) { callback_error ( { source : img . src , error : e } ) ; } ; img . src = images [ i ] ; this . img_cache [ images [ i ] ] = img ; this . preload_requests . push ( img ) ; } } preloadVideo ( videos , callback_complete = ( ) => { } , callback_load = ( filepath ) => { } , callback_error = ( error_msg ) => { } ) { videos = unique ( videos . flat ( ) ) ; let n_loaded = 0 ; if ( videos . length === 0 ) { callback_complete ( ) ; return ; } for ( const video of videos ) { const video_buffers = this . video_buffers ; const request = new XMLHttpRequest ( ) ; request . open ( "GET" , video , true ) ; request . responseType = "blob" ; request . onload = function ( ) { if ( this . status === 200 || this . status === 0 ) { const videoBlob = this . response ; video_buffers [ video ] = URL . createObjectURL ( videoBlob ) ; n_loaded ++ ; callback_load ( video ) ; if ( n_loaded === videos . length ) { callback_complete ( ) ; } } } ; request . onerror = function ( e ) { let err = e ; if ( this . status == 404 ) { err = "404" ; } callback_error ( { source : video , error : err } ) ; } ; request . onloadend = function ( e ) { if ( this . status == 404 ) { callback_error ( { source : video , error : "404" } ) ; } } ; request . send ( ) ; this . preload_requests . push ( request ) ; } } private preloadMap = new Map < string , Record < string , PreloadType > > ( ) ; getAutoPreloadList ( timeline_description ) { /** Map each preload parameter type to a set of paths to be preloaded */ const preloadPaths = Object . fromEntries ( preloadParameterTypes . map ( ( type ) => [ type , new Set < string > ( ) ] ) ) ; const traverseTimeline = ( node , inheritedTrialType ? ) => { const isTimeline = typeof node . timeline !== "undefined" ; if ( isTimeline ) { for ( const childNode of node . timeline ) { traverseTimeline ( childNode , node . type ?? inheritedTrialType ) ; } } else if ( ( node . type ?? inheritedTrialType ) ?. info ) { const { name : pluginName , parameters } = ( node . type ?? inheritedTrialType ) . info ; if ( ! this . preloadMap . has ( pluginName ) ) { this . preloadMap . set ( pluginName , Object . fromEntries ( Object . entries < any > ( parameters ) . filter ( ( [ _name , { type , preload } ] ) => preloadParameterTypes . includes ( type ) && ( preload ?? true ) ) . map ( ( [ name , { type } ] ) => [ name , type ] ) ) ) ; } for ( const [ parameterName , parameterType ] of Object . entries ( this . preloadMap . get ( pluginName ) ) ) { const parameterValue = node [ parameterName ] ; const elements = preloadPaths [ parameterType ] ; if ( typeof parameterValue === "string" ) { elements . add ( parameterValue ) ; } else if ( Array . isArray ( parameterValue ) ) { for ( const element of parameterValue . flat ( ) ) { if ( typeof element === "string" ) { elements . add ( element ) ; } } } } } } ; traverseTimeline ( { timeline : timeline_description } ) ; return { images : [ ... preloadPaths [ ParameterType . IMAGE ] ] , audio : [ ... preloadPaths [ ParameterType . AUDIO ] ] , video : [ ... preloadPaths [ ParameterType . VIDEO ] ] , } ; } cancelPreloads ( ) { for ( const request of this . preload_requests ) { request . onload = ( ) => { } ; request . onerror = ( ) => { } ; request . oncanplaythrough = ( ) => { } ; request . onabort = ( ) => { } ; } this . preload_requests = [ ] ; } private microphone_recorder : MediaRecorder = null ; initializeMicrophoneRecorder ( stream ) { const recorder = new MediaRecorder ( stream ) ; this . microphone_recorder = recorder ; } getMicrophoneRecorder ( ) { return this . microphone_recorder ; } }