import { TimeInstant } from '../time' ; import { Command } from 'nf-grapher' ; import { ScoreAudioParam } from '../params/ScoreAudioParam' ; import { SPNode , NodePlaybackDescription } from './SPNode' ; import { SPNodeFactory } from './SPNodeFactory' ; import { mixdown , asInterleaved , asPlanar } from '../AudioBufferUtils' ; import { ContentCache } from '../ContentCache' ; import { SoundTouch } from 'soundtouch-ts' ; import { CommandsMutation , applyMutationToParam } from '../Mutations' ; import { debug as Debug } from 'debug' ; import { XAudioBuffer } from '../XAudioBuffer' ; const DBG_STR = 'nf:stretchnode' ; const dbg = Debug ( DBG_STR ) ; let SHARED_SOUNDTOUCH_CTOR ; if ( typeof global !== 'undefined' && typeof ( global as any ) . SoundTouch !== 'undefined' ) { SHARED_SOUNDTOUCH_CTOR = ( global as any ) . SoundTouch ; } else if ( typeof window !== 'undefined' && typeof ( window as any ) . SoundTouch !== 'undefined' ) { SHARED_SOUNDTOUCH_CTOR = ( window as any ) . SoundTouch ; } else { SHARED_SOUNDTOUCH_CTOR = SoundTouch ; } let uniqId = 0 ; export class SPStretchNode extends SPNode { protected stretchParam = new ScoreAudioParam ( 1 ) ; protected pitchRatioParam = new ScoreAudioParam ( 1 ) ; protected previousFeedTime : TimeInstant = TimeInstant . ZERO ; protected requestedAncestorSamples : number = 0 ; protected st : SoundTouch = new SHARED_SOUNDTOUCH_CTOR ( this . info . sampleRate ) ; protected uniqId = uniqId ++ ; async nodeDidMount ( ) { this . stretchParam . applyScoreCommands ( this . node . params ! . stretch ) ; this . pitchRatioParam . applyScoreCommands ( this . node . params ! . pitchRatio ) ; } async nodeWillUnmount ( ) { dbg . enabled && dbg ( '%s %d unmounting. requestedAncestorSamples %f, previousFeed %f' , this . node . id , this . uniqId , this . requestedAncestorSamples , this . previousFeedTime . asSamples ( this . info . sampleRate ) ) ; } async timeChange ( renderTime , cache , quantumSize ) { const stretchValue = this . stretchParam . getValueAtTime ( renderTime . asSeconds ( ) ) ; const dilated = renderTime . scale ( stretchValue ) ; dbg . enabled && dbg ( '%s %d timeChange to %d, dilated %d. prev requested samples %d.' , this . node . id , this . uniqId , renderTime . asSamples ( this . info . sampleRate ) , dilated . asSamples ( this . info . sampleRate ) , this . requestedAncestorSamples ) ; await super . timeChange ( dilated , cache , quantumSize ) ; this . previousFeedTime = TimeInstant . from ( renderTime ) ; this . requestedAncestorSamples = this . estimateRequestedAncestorSamples ( renderTime , quantumSize ) ; dbg ( '%s %d new requested samples estimate: %d' , this . node . id , this . uniqId , this . requestedAncestorSamples ) ; this . st . clear ( ) ; this . st = new SHARED_SOUNDTOUCH_CTOR ( this . info . sampleRate ) ; } estimateRequestedAncestorSamples ( renderTime , quantumSize ) { const hz = this . info . sampleRate ; const renderTimeSamples = renderTime . asSamples ( hz ) ; const frameDuration = TimeInstant . fromSamples ( quantumSize , hz ) ; let receivedSamples = 0 ; let requiredSamples = 0 ; while ( receivedSamples <= renderTimeSamples ) { const frameStart = TimeInstant . fromSamples ( receivedSamples , hz ) ; const frameEnd = frameStart . add ( frameDuration ) ; const seconds = frameStart . asSeconds ( ) ; const frameEndSeconds = frameEnd . asSeconds ( ) ; const startStretchValue = this . stretchParam . getValueAtTime ( seconds ) ; const endStretchValue = this . stretchParam . getValueAtTime ( frameEndSeconds ) ; const averageStretchValue = ( endStretchValue + startStretchValue ) / 2 ; const startPitchValue = this . pitchRatioParam . getValueAtTime ( seconds ) ; const endPitchValue = this . pitchRatioParam . getValueAtTime ( frameEndSeconds ) ; const avgPitchValue = ( endPitchValue + startPitchValue ) / 2 ; const stVirtualPitch = avgPitchValue ; const stVirtualTempo = 1 / averageStretchValue ; const stVirtualRate = 1 ; const stTempo = stVirtualTempo / stVirtualPitch ; const stRate = stVirtualRate * stVirtualPitch ; const inputSamples = quantumSize * stTempo * stRate ; requiredSamples += inputSamples ; receivedSamples += quantumSize ; } return requiredSamples ; } getPlaybackDescription ( renderTime , descriptions ) { const ancestorRenderTime = TimeInstant . fromSamples ( this . requestedAncestorSamples , this . info . sampleRate ) ; const desc = { id : this . node . id , kind : this . node . kind , time : ancestorRenderTime } ; descriptions . push ( desc ) ; SPNodeFactory . getPlaybackDescription ( this . ancestors , ancestorRenderTime , descriptions ) ; } feed ( renderTime , buffers , sampleCount ) { const hz = this . info . sampleRate ; const seconds = renderTime . asSeconds ( ) ; const frameDuration = TimeInstant . fromSamples ( sampleCount , hz ) ; const frameEnd = renderTime . add ( frameDuration ) ; const frameEndSeconds = frameEnd . asSeconds ( ) ; const startStretchValue = this . stretchParam . getValueAtTime ( seconds ) ; const endStretchValue = this . stretchParam . getValueAtTime ( frameEndSeconds ) ; const averageStretchValue = ( endStretchValue + startStretchValue ) / 2 ; const startPitchValue = this . pitchRatioParam . getValueAtTime ( seconds ) ; const endPitchValue = this . pitchRatioParam . getValueAtTime ( frameEndSeconds ) ; const averagePitchValue = ( endPitchValue + startPitchValue ) / 2 ; const renderTimeSamples = renderTime . asSamples ( hz ) ; const previousFeedTimeSamples = this . previousFeedTime . asSamples ( hz ) ; if ( renderTimeSamples < previousFeedTimeSamples ) { const hz = this . info . sampleRate ; const prevFeed = this . previousFeedTime ; this . requestedAncestorSamples = this . estimateRequestedAncestorSamples ( renderTime , sampleCount ) ; dbg . enabled && dbg ( '%s %d prev feed time (%f) was _after_ current feed time (%f). ' + 'Estimating prev feed as %f and requested samples as %f' , this . node . id , this . uniqId , prevFeed . asSamples ( hz ) , renderTime . asSamples ( hz ) , this . previousFeedTime . asSamples ( hz ) , this . requestedAncestorSamples ) ; this . st . clear ( ) ; dbg ( '%s %d reset ancestor samples to %f' , this . node . id , this . uniqId , this . requestedAncestorSamples ) ; } const CHANNEL_COUNT = 2 ; const output = new Float32Array ( sampleCount * CHANNEL_COUNT ) ; this . st . pitch = averagePitchValue ; this . st . tempo = 1 / averageStretchValue ; let receivedSamples = 0 ; const requiredSampleCount = sampleCount * CHANNEL_COUNT ; while ( receivedSamples < requiredSampleCount ) { this . st . process ( ) ; const queuedIFrames = this . st . outputBuffer . frameCount ; const iFrameCount = sampleCount / CHANNEL_COUNT ; const receiver = output . subarray ( receivedSamples ) ; const requestedFrameCount = Math . min ( Math . floor ( iFrameCount ) , Math . floor ( receiver . length / CHANNEL_COUNT ) ) ; this . st . outputBuffer . receiveSamples ( receiver , requestedFrameCount ) ; const remainingIFrames = this . st . outputBuffer . frameCount ; const receivedThisStep = ( queuedIFrames - remainingIFrames ) * CHANNEL_COUNT ; receivedSamples += receivedThisStep ; if ( receivedThisStep > 0 ) { continue ; } const ancestorScratch = new XAudioBuffer ( { numberOfChannels : CHANNEL_COUNT , length : sampleCount , sampleRate : hz } ) ; const frameStartSamples = this . requestedAncestorSamples ; const ancestorBuffers = [ ] ; SPNodeFactory . feed ( this . ancestors , TimeInstant . fromSamples ( frameStartSamples , hz ) , ancestorBuffers , sampleCount ) ; mixdown ( ancestorScratch , ancestorBuffers ) ; this . requestedAncestorSamples += sampleCount ; const interleaved = asInterleaved ( ancestorScratch ) ; this . st . inputBuffer . putSamples ( interleaved ) ; } const planar = asPlanar ( output , this . info . sampleRate , CHANNEL_COUNT ) ; buffers . push ( planar ) ; this . previousFeedTime = TimeInstant . from ( renderTime ) ; } acceptCommandsEffect ( effect ) { const { paramName } = effect ; let param = null ; let gparam = null ; switch ( paramName ) { case 'stretch' : { param = this . stretchParam ; gparam = this . node . params ! . stretch ; break ; } case 'pitchRatio' : { param = this . pitchRatioParam ; gparam = this . node . params ! . pitchRatio ; break ; } case 'formantRatio' : { gparam = this . node . params ! . formantRatio ; break ; } } if ( ! param || ! gparam ) { throw new Error ( `Unknown or Unimplemented Param with name ${ paramName }` ) ; } return applyMutationToParam ( effect , param , gparam ) ; } }