import { Token } from "./parser/tokenizer" ; import { ContextualKeyword } from "./parser/tokenizer/keywords" ; import { TokenType , TokenType as tt } from "./parser/tokenizer/types" ; class StringBuilder { private buffer : ArrayBuffer = new ArrayBuffer ( 2 ** 24 ) ; private view : DataView = new DataView ( this . buffer ) ; private offset : i32 = 0 ; append ( str ) { const mem = String . UTF16 . encode ( str ) ; const view = new DataView ( mem ) ; for ( let i = 0 ; i < view . byteLength ; i ++ ) { this . view . setUint8 ( this . offset + i , view . getUint8 ( i ) ) ; } this . offset += view . byteLength ; } getString ( ) { return String . UTF16 . decode ( this . buffer . slice ( 0 , this . offset ) ) ; } } export default class TokenProcessor { private builder : StringBuilder = new StringBuilder ( ) ; private resultCode ( out ) { this . builder . append ( out ) ; } private tokenIndex : i32 = 0 ; constructor ( readonly code , readonly tokens ) { } reset ( ) { this . builder = new StringBuilder ( ) ; this . tokenIndex = 0 ; } matchesContextualAtIndex ( index , contextualKeyword ) { return ( this . matches1AtIndex ( index , tt . name ) && this . tokens [ index ] . contextualKeyword === contextualKeyword ) ; } identifierNameAtIndex ( index ) { return this . identifierNameForToken ( this . tokens [ index ] ) ; } identifierName ( ) { return this . identifierNameForToken ( this . currentToken ( ) ) ; } identifierNameForToken ( token ) { return this . code . slice ( token . start , token . end ) ; } stringValueForToken ( token ) { return this . code . slice ( token . start + 1 , token . end - 1 ) ; } matches1AtIndex ( index , t1 ) { return this . tokens [ index ] . type === t1 ; } matches1 ( t1 ) { return this . tokens [ this . tokenIndex ] . type === t1 ; } matches2 ( t1 , t2 ) { return this . tokens [ this . tokenIndex ] . type === t1 && this . tokens [ this . tokenIndex + 1 ] . type === t2 ; } matches3 ( t1 , t2 , t3 ) { return ( this . tokens [ this . tokenIndex ] . type === t1 && this . tokens [ this . tokenIndex + 1 ] . type === t2 && this . tokens [ this . tokenIndex + 2 ] . type === t3 ) ; } matchesContextual ( contextualKeyword ) { return this . matchesContextualAtIndex ( this . tokenIndex , contextualKeyword ) ; } previousWhitespaceAndComments ( ) { let whitespaceAndComments = this . code . slice ( this . tokenIndex > 0 ? this . tokens [ this . tokenIndex - 1 ] . end : 0 , this . tokenIndex < this . tokens . length ? this . tokens [ this . tokenIndex ] . start : this . code . length ) ; return whitespaceAndComments ; } replaceToken ( newCode ) { this . resultCode ( this . previousWhitespaceAndComments ( ) ) ; this . appendTokenPrefix ( ) ; this . resultCode ( newCode ) ; this . appendTokenSuffix ( ) ; this . tokenIndex ++ ; } replaceTokenTrimmingLeftWhitespace ( newCode ) { this . appendTokenPrefix ( ) ; this . resultCode ( newCode ) ; this . appendTokenSuffix ( ) ; this . tokenIndex ++ ; } removeInitialToken ( ) { this . replaceToken ( "" ) ; } removeToken ( ) { this . replaceTokenTrimmingLeftWhitespace ( "" ) ; } copyExpectedToken ( tokenType ) { if ( this . tokens [ this . tokenIndex ] . type !== tokenType ) { throw new Error ( `Expected token ${ tokenType }` ) ; } this . copyToken ( ) ; } copyToken ( ) { this . resultCode ( this . previousWhitespaceAndComments ( ) ) ; this . appendTokenPrefix ( ) ; this . resultCode ( this . code . slice ( this . tokens [ this . tokenIndex ] . start , this . tokens [ this . tokenIndex ] . end ) ) ; this . appendTokenSuffix ( ) ; this . tokenIndex ++ ; } private appendTokenPrefix ( ) { } private appendTokenSuffix ( ) { } appendCode ( code ) { this . resultCode ( code ) ; } currentToken ( ) { return this . tokens [ this . tokenIndex ] ; } tokenAtRelativeIndex ( relativeIndex ) { return this . tokens [ this . tokenIndex + relativeIndex ] ; } currentIndex ( ) { return this . tokenIndex ; } finish ( ) { if ( this . tokenIndex !== this . tokens . length ) { throw new Error ( "Tried to finish processing tokens before reaching the end." ) ; } this . resultCode ( this . previousWhitespaceAndComments ( ) ) ; return this . builder . getString ( ) ; } isAtEnd ( ) { return this . tokenIndex === this . tokens . length ; } }