import { TokenPosition , ParseError } from './util' ; type TokenContent = { type : 'literal' ; value : string ; } | { type : 'value' ; value : unknown ; } | { type : 'identifier' ; value : string ; } ; export type Token = TokenContent & TokenPosition ; abstract class LexerError { message : string ; constructor ( public pos ) { } } class NoTokenMatchError extends LexerError { message = 'No match for token' ; } class StringNewlineError extends LexerError { message = 'Unexpected newline in string' ; } class StringIncompleteError extends LexerError { message = 'Unexpected end of input in string' ; } class LexerState { index = 0 ; outerIndex = 0 ; private tokens : Token [ ] = [ ] ; constructor ( public readonly strings , public readonly interps ) { } getTokens ( ) { return this . tokens ; } push ( token ) { if ( ! token ) return ; this . tokens . push ( token ) ; } getInterpolation ( ) { let token ; if ( this . outerIndex < this . interps . length ) { token = { type : 'value' , value : this . interps [ this . outerIndex ] , index : 0 , outerIndex : this . outerIndex + 1 , length : 0 , } ; } this . outerIndex ++ ; this . index = 0 ; return token ; } hasStrings ( ) { return this . outerIndex < this . strings . length ; } nextChar ( ) { const buf = this . strings [ this . outerIndex ] ; if ( ! buf ) return undefined ; return buf [ this . index ] ; } matchPattern ( pattern ) { const buf = this . strings [ this . outerIndex ] ; pattern . lastIndex = this . index ; const result = pattern . exec ( buf ) ; if ( ! result ) return null ; this . index = pattern . lastIndex ; return result ; } } export const identifierPattern = /(?:\$|_|\p{ID_Start})(?:\$|\u200C|\u200D|\p{ID_Continue})*/u ; const singleQuotePattern = /((?:\\['\\]|[^\n'\\])+)|(')/y ; const doubleQuotePattern = /((?:\\["\\]|[^\n"\\])+)|(")/y ; const lineCommentPattern = /([^\n]+)|(\n)/y ; const blockCommentPattern = /((?:\*[^/]|[^*])+)|(\*\/)/y ; export const identifierOrOperator = /^(?:(?:\$|_|\p{ID_Start})(?:\$|\u200C|\u200D|\p{ID_Continue})*|[!@#%^&*\-+=|/:<>.?~]+|[,;])$/u ; export class Lexer { lexerState : LexerState ; mainPattern : RegExp ; constructor ( private readonly keywords , operators ) { const operatorsPattern = matchOperators ( Array . from ( operators ) ) ; this . mainPattern = new RegExp ( [ /[ \t\n]+/ , /\/\// , /\/\*/ , /0x[0-9A-Fa-f_]+/ , /0o[0-7_]+/ , /0b[0-1_]+/ , /-?[0-9][0-9_]*(?:\.[0-9_]*)?(?:[eE]-?[0-9_])?/ , identifierPattern , operatorsPattern , ] . map ( re => `(${ re . source })` ) . join ( '|' ) , 'uy' ) ; } run ( strs , interps ) { this . lexerState = new LexerState ( strs , interps ) ; try { this . mainState ( ) ; } catch ( e ) { if ( e instanceof LexerError ) { throw new ParseError ( e . message , strs , e . pos ) ; } else { throw e ; } } return this . lexerState . getTokens ( ) ; } private mainState ( ) { const lexerState = this . lexerState ; let ch ; while ( lexerState . hasStrings ( ) ) { while ( ( ch = lexerState . nextChar ( ) ) ) { switch ( ch ) { case `'` : lexerState . index ++ ; this . quote ( singleQuotePattern ) ; continue ; case `"` : lexerState . index ++ ; this . quote ( doubleQuotePattern ) ; continue ; } const lastIndex = lexerState . index ; const match = lexerState . matchPattern ( this . mainPattern ) ; if ( ! match ) { throw new NoTokenMatchError ( { index : lexerState . index , outerIndex : lexerState . outerIndex , length : 1 , } ) ; } const matchedString = match [ 0 ] ; if ( match [ 2 ] ) { this . comment ( lineCommentPattern ) ; } else if ( match [ 3 ] ) { this . comment ( blockCommentPattern ) ; } else if ( match [ 4 ] || match [ 5 ] || match [ 6 ] || match [ 7 ] ) { const value = Number ( matchedString ) ; lexerState . push ( { type : 'value' , value , index : lastIndex , outerIndex : lexerState . outerIndex , length : matchedString . length , } ) ; } else if ( match [ 8 ] ) { lexerState . push ( { type : this . keywords . has ( matchedString ) ? 'literal' : 'identifier' , value : matchedString , index : lastIndex , outerIndex : lexerState . outerIndex , length : matchedString . length , } ) ; } else if ( match [ 9 ] ) { lexerState . push ( { type : 'literal' , value : matchedString , index : lastIndex , outerIndex : lexerState . outerIndex , length : matchedString . length , } ) ; } } lexerState . push ( lexerState . getInterpolation ( ) ) ; } } private quote ( pattern ) { const lexerState = this . lexerState ; const token = { type : 'value' , value : '' , index : lexerState . index - 1 , outerIndex : lexerState . outerIndex , length : 1 , } ; while ( lexerState . hasStrings ( ) ) { while ( lexerState . nextChar ( ) ) { const match = lexerState . matchPattern ( pattern ) ; if ( ! match ) throw new StringNewlineError ( token ) ; if ( match [ 1 ] ) { token . value += match [ 1 ] ; token . length += match [ 1 ] . length ; } else { token . length ++ ; lexerState . push ( token ) ; return ; } } const interpolatedToken = lexerState . getInterpolation ( ) ; if ( interpolatedToken ) { token . value += String ( interpolatedToken . value ) ; } } throw new StringIncompleteError ( token ) ; } private comment ( pattern ) { while ( this . lexerState . hasStrings ( ) ) { while ( this . lexerState . nextChar ( ) ) { const match = this . lexerState . matchPattern ( pattern ) ; if ( ! match || match [ 2 ] ) return ; } this . lexerState . getInterpolation ( ) ; } } } function reEscape ( s ) { return s . replace ( /[/\\^$*+?.()|[\]{}]/g , '\\$&' ) ; } function matchOperators ( operators ) { const longestFirst = Array . from ( new Set ( operators ) ) . sort ( ( a , b ) => b . length - a . length ) ; return new RegExp ( `(?:${ longestFirst . map ( reEscape ) . join ( '|' ) })` ) ; }