import assert from "assert" ; import { TestThrottlerHelper } from "@fluidframework/server-test-utils" ; import { Throttler } from "../throttler" ; import { IThrottler , ThrottlingError } from "@fluidframework/server-services-core" ; import { TestEngine1 , Lumberjack } from "@fluidframework/server-services-telemetry" ; import Sinon from "sinon" ; const lumberjackEngine = new TestEngine1 ( ) ; if ( ! Lumberjack . isSetupCompleted ( ) ) { Lumberjack . setup ( [ lumberjackEngine ] ) ; } describe ( "Throttler" , ( ) => { beforeEach ( ( ) => { Sinon . useFakeTimers ( Date . now ( ) ) ; } ) ; afterEach ( ( ) => { Sinon . restore ( ) ; } ) ; const exceedThrottleLimit = async ( id , throttler , rate , minThrottleCheckInterval , numIntervalsToBeThrottledFor = 1 , weight = 1 ) => { const numOperationsToExceedIntervalLimit = Math . ceil ( minThrottleCheckInterval / rate ) ; const numOperations = numIntervalsToBeThrottledFor * numOperationsToExceedIntervalLimit ; for ( let i = 0 ; i < numOperations + 1 ; i ++ ) { await Sinon . clock . nextAsync ( ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id , weight ) ; } ) ; } } ; it ( "does not throttle within min throttle check interval" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval ) ; const id = "test1" ; await exceedThrottleLimit ( id , throttler , rate , minThrottleCheckInterval , 1 ) ; } ) ; it ( "throttles after min throttle check interval" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval ) ; const id = "test2" ; await exceedThrottleLimit ( id , throttler , rate , minThrottleCheckInterval , 1 ) ; Sinon . clock . tick ( minThrottleCheckInterval + 1 ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id , 0 ) ; } ) ; await Sinon . clock . nextAsync ( ) ; assert . throws ( ( ) => { throttler . incrementCount ( id ) ; } , ThrottlingError ) ; } ) ; it ( "throttles after multiple throttle check intervals" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval ) ; const id = "test3" ; await exceedThrottleLimit ( id , throttler , rate , minThrottleCheckInterval , 2 ) ; Sinon . clock . tick ( minThrottleCheckInterval * 2 + 1 ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id , 0 ) ; } ) ; await Sinon . clock . nextAsync ( ) ; assert . throws ( ( ) => { throttler . incrementCount ( id ) ; } , ThrottlingError ) ; } ) ; it ( "throttles fewer heavier operations after min throttle check interval" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval ) ; const id = "test2" ; await exceedThrottleLimit ( id , throttler , rate , minThrottleCheckInterval , 0.5 , 2 ) ; Sinon . clock . tick ( minThrottleCheckInterval + 1 ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id , 0 ) ; } ) ; await Sinon . clock . nextAsync ( ) ; assert . throws ( ( ) => { throttler . incrementCount ( id ) ; } , ThrottlingError ) ; } ) ; it ( "un-throttles early if operations are closed" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval ) ; const id = "test4" ; await exceedThrottleLimit ( id , throttler , rate , minThrottleCheckInterval , 2 ) ; const limit = minThrottleCheckInterval / rate ; for ( let i = 0 ; i < limit + 1 ; i ++ ) { throttler . decrementCount ( id ) ; } Sinon . clock . tick ( minThrottleCheckInterval * 2 + 1 ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id , 0 ) ; } ) ; await Sinon . clock . nextAsync ( ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id , 0 ) ; } ) ; } ) ; it ( "un-throttles fewer heavier operations early if operations are closed" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval ) ; const id = "test4" ; await exceedThrottleLimit ( id , throttler , rate , minThrottleCheckInterval , 1 , 2 ) ; const limit = minThrottleCheckInterval / rate ; for ( let i = 0 ; i < limit / 2 + 1 ; i ++ ) { throttler . decrementCount ( id , 2 ) ; } Sinon . clock . tick ( minThrottleCheckInterval * 2 + 1 ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id , 0 ) ; } ) ; await Sinon . clock . nextAsync ( ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id , 0 ) ; } ) ; } ) ; it ( "does not throttle when Throttler fails to update throttle status" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; Sinon . stub ( throttlerHelper , "updateCount" ) . rejects ( ) ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval ) ; const id = "test5" ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id ) ; } ) ; await Sinon . clock . nextAsync ( ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id ) ; } ) ; } ) ; it ( "does not throttle when Throttler fails to retrieve throttle status" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; Sinon . stub ( throttlerHelper , "getThrottleStatus" ) . rejects ( ) ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval ) ; const id = "test6" ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id ) ; } ) ; await Sinon . clock . nextAsync ( ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id ) ; } ) ; } ) ; it ( "leniently throttles an un-cached, throttled operation" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; Sinon . stub ( throttlerHelper , "updateCount" ) . resolves ( { throttleStatus : true , throttleReason : "Exceeded count" , retryAfterInMs : 100 } ) ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval ) ; const id = "test7" ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id ) ; } ) ; Sinon . clock . tick ( minThrottleCheckInterval * 2 + 1 ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id , 0 ) ; } ) ; await Sinon . clock . nextAsync ( ) ; assert . throws ( ( ) => { throttler . incrementCount ( id ) ; } , ThrottlingError ) ; } ) ; it ( "does not cache too many operation ids" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; Sinon . stub ( throttlerHelper , "updateCount" ) . resolves ( { throttleStatus : true , throttleReason : "Exceeded count" , retryAfterInMs : 100 } ) ; const cacheSize = 5 ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval , undefined , cacheSize ) ; for ( let i = 0 ; i < cacheSize ; i ++ ) { throttler . incrementCount ( `test${ i }` ) ; } Sinon . clock . tick ( minThrottleCheckInterval + 1 ) ; for ( let i = 0 ; i < cacheSize ; i ++ ) { assert . doesNotThrow ( ( ) => { throttler . incrementCount ( `test${ i }` ) ; } ) ; } await Sinon . clock . nextAsync ( ) ; for ( let i = 0 ; i < cacheSize ; i ++ ) { assert . throws ( ( ) => { throttler . incrementCount ( `test${ i }` ) ; } , ThrottlingError ) ; } for ( let i = 0 ; i < cacheSize ; i ++ ) { throttler . incrementCount ( `test${ i + cacheSize }` ) ; } Sinon . clock . tick ( minThrottleCheckInterval + 1 ) ; for ( let i = 0 ; i < cacheSize ; i ++ ) { assert . doesNotThrow ( ( ) => { throttler . incrementCount ( `test${ i + cacheSize }` ) ; } ) ; } await Sinon . clock . nextAsync ( ) ; for ( let i = 0 ; i < cacheSize ; i ++ ) { assert . throws ( ( ) => { throttler . incrementCount ( `test${ i + cacheSize }` ) ; } , ThrottlingError ) ; } for ( let i = 0 ; i < cacheSize ; i ++ ) { assert . doesNotThrow ( ( ) => { throttler . incrementCount ( `test${ i }` ) ; } ) ; } } ) ; it ( "expires old cached values" , async ( ) => { const rate = 10 ; const minThrottleCheckInterval = 100 ; const throttlerHelper = new TestThrottlerHelper ( rate ) ; Sinon . stub ( throttlerHelper , "updateCount" ) . resolves ( { throttleStatus : true , throttleReason : "Exceeded count" , retryAfterInMs : 100 } ) ; const cacheAge = 5000 ; const throttler = new Throttler ( throttlerHelper , minThrottleCheckInterval , undefined , undefined , cacheAge ) ; const id = "testCache" ; await throttler . incrementCount ( id ) ; Sinon . clock . tick ( minThrottleCheckInterval + 1 ) ; await throttler . incrementCount ( id ) ; await Sinon . clock . nextAsync ( ) ; assert . throws ( ( ) => { throttler . incrementCount ( id ) ; } , ThrottlingError ) ; Sinon . clock . tick ( cacheAge + 1 ) ; assert . doesNotThrow ( ( ) => { throttler . incrementCount ( id ) ; } ) ; } ) ; } ) ;