license import { env , Tensor , tensor , Tensor2D , Tensor3D , TensorContainer , util } from '@tensorflow/tfjs-core' ; import { MicrophoneConfig } from '../types' ; import { LazyIterator } from './lazy_iterator' ; /**
 * Provide a stream of tensors from microphone audio stream. The tensors are
 * representing audio data as frequency-domain spectrogram generated with
 * browser's native FFT. Tensors representing time-domain waveform is available
 * based on configuration. Only works in browser environment.
 */ export class MicrophoneIterator extends LazyIterator < TensorContainer > { private isClosed = false ; private stream : MediaStream ; private readonly fftSize : number ; private readonly columnTruncateLength : number ; private freqData : Float32Array ; private timeData : Float32Array ; private readonly numFrames : number ; private analyser : AnalyserNode ; private audioContext : AudioContext ; private sampleRateHz : number ; private readonly audioTrackConstraints : MediaTrackConstraints ; private readonly smoothingTimeConstant : number ; private readonly includeSpectrogram : boolean ; private readonly includeWaveform : boolean ; private constructor ( protected readonly microphoneConfig ) { super ( ) ; this . fftSize = microphoneConfig . fftSize || 1024 ; const fftSizeLog2 = Math . log2 ( this . fftSize ) ; if ( this . fftSize < 0 || fftSizeLog2 < 4 || fftSizeLog2 > 14 || ! Number . isInteger ( fftSizeLog2 ) ) { throw new Error ( `Invalid fftSize: it must be a power of 2 between ` + `2 to 4 and 2 to 14, but got ${ this . fftSize }` ) ; } this . numFrames = microphoneConfig . numFramesPerSpectrogram || 43 ; this . sampleRateHz = microphoneConfig . sampleRateHz ; this . columnTruncateLength = microphoneConfig . columnTruncateLength || this . fftSize ; this . audioTrackConstraints = microphoneConfig . audioTrackConstraints ; this . smoothingTimeConstant = microphoneConfig . smoothingTimeConstant || 0 ; this . includeSpectrogram = microphoneConfig . includeSpectrogram === false ? false : true ; this . includeWaveform = microphoneConfig . includeWaveform === true ? true : false ; if ( ! this . includeSpectrogram && ! this . includeWaveform ) { throw new Error ( 'Both includeSpectrogram and includeWaveform are false. ' + 'At least one type of data should be returned.' ) ; } } summary ( ) { return `microphone` ; } static async create ( microphoneConfig = { } ) { if ( env ( ) . get ( 'IS_NODE' ) ) { throw new Error ( 'microphone API is only supported in browser environment.' ) ; } const microphoneIterator = new MicrophoneIterator ( microphoneConfig ) ; await microphoneIterator . start ( ) ; return microphoneIterator ; } async start ( ) { try { this . stream = await navigator . mediaDevices . getUserMedia ( { audio : this . audioTrackConstraints == null ? true : this . audioTrackConstraints , video : false } ) ; } catch ( e ) { throw new Error ( `Error thrown while initializing video stream: ${ e . message }` ) ; } if ( ! this . stream ) { throw new Error ( 'Could not obtain audio from microphone.' ) ; } const ctxConstructor = ( window as any ) . AudioContext || ( window as any ) . webkitAudioContext ; this . audioContext = new ctxConstructor ( ) ; if ( ! this . sampleRateHz ) { this . sampleRateHz = this . audioContext . sampleRate ; } else if ( this . audioContext . sampleRate !== this . sampleRateHz ) { throw new Error ( `Mismatch in sampling rate: ` + `Expected: ${ this . sampleRateHz }; ` + `Actual: ${ this . audioContext . sampleRate }` ) ; } const streamSource = this . audioContext . createMediaStreamSource ( this . stream ) ; this . analyser = this . audioContext . createAnalyser ( ) ; this . analyser . fftSize = this . fftSize * 2 ; this . analyser . smoothingTimeConstant = this . smoothingTimeConstant ; streamSource . connect ( this . analyser ) ; this . freqData = new Float32Array ( this . fftSize ) ; this . timeData = new Float32Array ( this . fftSize ) ; return ; } async next ( ) { if ( this . isClosed ) { return { value : null , done : true } ; } let spectrogramTensor ; let waveformTensor ; const audioDataQueue = await this . getAudioData ( ) ; if ( this . includeSpectrogram ) { const freqData = this . flattenQueue ( audioDataQueue . freqDataQueue ) ; spectrogramTensor = this . getTensorFromAudioDataArray ( freqData , [ this . numFrames , this . columnTruncateLength , 1 ] ) ; } if ( this . includeWaveform ) { const timeData = this . flattenQueue ( audioDataQueue . timeDataQueue ) ; waveformTensor = this . getTensorFromAudioDataArray ( timeData , [ this . numFrames * this . fftSize , 1 ] ) ; } return { value : { 'spectrogram' : spectrogramTensor , 'waveform' : waveformTensor } , done : false } ; } async capture ( ) { return ( await this . next ( ) ) . value as { spectrogram : Tensor3D ; waveform : Tensor2D ; } ; } private async getAudioData ( ) { const freqDataQueue = [ ] ; const timeDataQueue = [ ] ; let currentFrames = 0 ; return new Promise ( resolve => { const intervalID = setInterval ( ( ) => { if ( this . includeSpectrogram ) { this . analyser . getFloatFrequencyData ( this . freqData ) ; if ( this . freqData [ 0 ] === - Infinity ) { resolve ( { freqDataQueue , timeDataQueue } ) ; } freqDataQueue . push ( this . freqData . slice ( 0 , this . columnTruncateLength ) ) ; } if ( this . includeWaveform ) { this . analyser . getFloatTimeDomainData ( this . timeData ) ; timeDataQueue . push ( this . timeData . slice ( ) ) ; } if ( ++ currentFrames === this . numFrames ) { clearInterval ( intervalID ) ; resolve ( { freqDataQueue , timeDataQueue } ) ; } } , this . fftSize / this . sampleRateHz * 1e3 ) ; } ) ; } stop ( ) { if ( ! this . isClosed ) { this . isClosed = true ; this . analyser . disconnect ( ) ; this . audioContext . close ( ) ; if ( this . stream != null && this . stream . getTracks ( ) . length > 0 ) { this . stream . getTracks ( ) [ 0 ] . stop ( ) ; } } } toArray ( ) { throw new Error ( 'Can not convert infinite audio stream to array.' ) ; } getSampleRate ( ) { return this . sampleRateHz ; } private flattenQueue ( queue ) { const frameSize = queue [ 0 ] . length ; const freqData = new Float32Array ( queue . length * frameSize ) ; queue . forEach ( ( data , i ) => freqData . set ( data , i * frameSize ) ) ; return freqData ; } private getTensorFromAudioDataArray ( freqData , shape ) { const vals = new Float32Array ( util . sizeFromShape ( shape ) ) ; vals . set ( freqData , vals . length - freqData . length ) ; return tensor ( vals , shape ) ; } }