import escapeRegExp from '../core/escapeRegExp' ; import { TokenTypes , Token , TokenizerConfig } from './types' ; export default class Tokenizer { public WHITESPACE_REGEX : RegExp ; public NUMBER_REGEX : RegExp ; public AMBIGUOS_OPERATOR_REGEX : RegExp ; public OPERATOR_REGEX : RegExp ; public NO_SPACE_OPERATOR_REGEX : RegExp ; public BLOCK_COMMENT_REGEX : RegExp ; public LINE_COMMENT_REGEX : RegExp ; public RESERVED_TOP_LEVEL_REGEX : RegExp ; public RESERVED_TOP_LEVEL_NO_INDENT_REGEX : RegExp ; public RESERVED_NEWLINE_REGEX : RegExp ; public RESERVED_PLAIN_REGEX : RegExp ; public WORD_REGEX : RegExp ; public STRING_REGEX : RegExp ; public OPEN_PAREN_REGEX : RegExp ; public CLOSE_PAREN_REGEX : RegExp ; public INDEXED_PLACEHOLDER_REGEX : RegExp ; public IDENT_NAMED_PLACEHOLDER_REGEX : RegExp ; public STRING_NAMED_PLACEHOLDER_REGEX : RegExp ; param { TokenizerConfig } cfg param { string [ ] } cfg . reservedWords param { string [ ] } cfg . reservedTopLevelWords param { string [ ] } cfg . reservedNewlineWords param { string [ ] } cfg . reservedTopLevelWordsNoIndent param { string [ ] } cfg . stringTypes param { string [ ] } cfg . openParens param { string [ ] } cfg . closeParens param { string [ ] } cfg . indexedPlaceholderTypes param { string [ ] } cfg . namedPlaceholderTypes param { string [ ] } cfg . lineCommentTypes param { string [ ] } cfg . specialWordChars constructor ( cfg ) { this . WHITESPACE_REGEX = /^(\s+)/u ; this . NUMBER_REGEX = /^((-\s*)?[0-9]+(\.[0-9]+)?|0x[0-9a-fA-F]+|0b[01]+|([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}))\b/u ; this . AMBIGUOS_OPERATOR_REGEX = /^(\?\||\?&)/u ; this . OPERATOR_REGEX = /^(!=|<>|>>|<<|==|<=|>=|!<|!>|\|\|\/|\|\/|\|\||~~\*|~~|!~~\*|!~~|~\*|!~\*|!~|:=|&&|@>|<@|#-|@|.)/u ; this . NO_SPACE_OPERATOR_REGEX = /^(::|->>|->|#>>|#>)/u ; this . BLOCK_COMMENT_REGEX = /^(\/\*[^]*?(?:\*\/|$))/u ; this . LINE_COMMENT_REGEX = this . createLineCommentRegex ( cfg . lineCommentTypes ) ; this . RESERVED_TOP_LEVEL_REGEX = this . createReservedWordRegex ( cfg . reservedTopLevelWords ) ; this . RESERVED_TOP_LEVEL_NO_INDENT_REGEX = this . createReservedWordRegex ( cfg . reservedTopLevelWordsNoIndent ) ; this . RESERVED_NEWLINE_REGEX = this . createReservedWordRegex ( cfg . reservedNewlineWords ) ; this . RESERVED_PLAIN_REGEX = this . createReservedWordRegex ( cfg . reservedWords ) ; this . WORD_REGEX = this . createWordRegex ( cfg . specialWordChars ) ; this . STRING_REGEX = this . createStringRegex ( cfg . stringTypes ) ; this . OPEN_PAREN_REGEX = this . createParenRegex ( cfg . openParens ) ; this . CLOSE_PAREN_REGEX = this . createParenRegex ( cfg . closeParens ) ; this . INDEXED_PLACEHOLDER_REGEX = this . createPlaceholderRegex ( cfg . indexedPlaceholderTypes , '[0-9]*' ) ; this . IDENT_NAMED_PLACEHOLDER_REGEX = this . createPlaceholderRegex ( cfg . namedPlaceholderTypes , '[a-zA-Z0-9._$]+' ) ; this . STRING_NAMED_PLACEHOLDER_REGEX = this . createPlaceholderRegex ( cfg . namedPlaceholderTypes , this . createStringPattern ( cfg . stringTypes ) ) ; } createLineCommentRegex ( lineCommentTypes ) { return new RegExp ( `^((?:${ lineCommentTypes . map ( c => escapeRegExp ( c ) ) . join ( '|' ) })[^>]*?(?:\r\n|\r|\n|$))` , 'u' ) ; } createReservedWordRegex ( reservedWords ) { const reservedWordsPattern = reservedWords . join ( '|' ) . replace ( / /gu , '\\s+' ) ; return new RegExp ( `^(${ reservedWordsPattern })\\b` , 'iu' ) ; } createWordRegex ( specialChars ) { return new RegExp ( `^([\\p{Alphabetic}\\p{Mark}\\p{Decimal_Number}\\p{Connector_Punctuation}\\p{Join_Control}${ specialChars . join ( '' ) }]+)` , 'u' ) ; } createStringRegex ( stringTypes ) { return new RegExp ( '^(' + this . createStringPattern ( stringTypes ) + ')' , 'u' ) ; } createStringPattern ( stringTypes ) { const patterns = { '``' : '((`[^`]*($|`))+)' , '[]' : '((\\[[^\\]]*($|\\]))(\\][^\\]]*($|\\]))*)' , '""' : '(("[^"\\\\]*(?:\\\\.[^"\\\\]*)*("|$))+)' , "''" : "(('[^'\\\\]*(?:\\\\.[^'\\\\]*)*('|$))+)" , "N''" : "((N'[^N'\\\\]*(?:\\\\.[^N'\\\\]*)*('|$))+)" } ; return stringTypes . map ( t => patterns [ t ] ) . join ( '|' ) ; } createParenRegex ( parens ) { return new RegExp ( '^(' + parens . map ( p => this . escapeParen ( p ) ) . join ( '|' ) + ')' , 'iu' ) ; } escapeParen ( paren ) { if ( paren . length === 1 ) { return escapeRegExp ( paren ) ; } else { return '\\b' + paren + '\\b' ; } } createPlaceholderRegex ( types , pattern ) { if ( ! types || types . length === 0 ) { return null ; } const typesRegex = types . map ( escapeRegExp ) . join ( '|' ) ; return new RegExp ( `^((?:${ typesRegex })(?:${ pattern }))` , 'u' ) ; } param { string } input return { Object [ ] } return { string } return { string } tokenize ( input ) { if ( ! input ) return [ ] ; const tokens = [ ] ; let token ; while ( input . length ) { token = this . getNextToken ( input , token ) ; input = input . substring ( token . value . length ) ; tokens . push ( token ) ; } return tokens ; } getNextToken ( input , previousToken ? ) { return ( this . getWhitespaceToken ( input ) || this . getCommentToken ( input ) || this . getStringToken ( input ) || this . getOpenParenToken ( input ) || this . getCloseParenToken ( input ) || this . getAmbiguosOperatorToken ( input ) || this . getNoSpaceOperatorToken ( input ) || this . getServerVariableToken ( input ) || this . getPlaceholderToken ( input ) || this . getNumberToken ( input ) || this . getReservedWordToken ( input , previousToken ) || this . getWordToken ( input ) || this . getOperatorToken ( input ) ) ; } getWhitespaceToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . WHITESPACE , regex : this . WHITESPACE_REGEX } ) ; } getCommentToken ( input ) { return this . getLineCommentToken ( input ) || this . getBlockCommentToken ( input ) ; } getLineCommentToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . LINE_COMMENT , regex : this . LINE_COMMENT_REGEX } ) ; } getBlockCommentToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . BLOCK_COMMENT , regex : this . BLOCK_COMMENT_REGEX } ) ; } getStringToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . STRING , regex : this . STRING_REGEX } ) ; } getOpenParenToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . OPEN_PAREN , regex : this . OPEN_PAREN_REGEX } ) ; } getCloseParenToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . CLOSE_PAREN , regex : this . CLOSE_PAREN_REGEX } ) ; } getPlaceholderToken ( input ) { return ( this . getIdentNamedPlaceholderToken ( input ) || this . getStringNamedPlaceholderToken ( input ) || this . getIndexedPlaceholderToken ( input ) ) ; } getServerVariableToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . SERVERVARIABLE , regex : /(^@@\w+)/iu } ) ; } getIdentNamedPlaceholderToken ( input ) { return this . getPlaceholderTokenWithKey ( { input , regex : this . IDENT_NAMED_PLACEHOLDER_REGEX , parseKey : v => v . slice ( 1 ) } ) ; } getStringNamedPlaceholderToken ( input ) { return this . getPlaceholderTokenWithKey ( { input , regex : this . STRING_NAMED_PLACEHOLDER_REGEX , parseKey : v => this . getEscapedPlaceholderKey ( { key : v . slice ( 2 , - 1 ) , quoteChar : v . slice ( - 1 ) } ) } ) ; } getIndexedPlaceholderToken ( input ) { return this . getPlaceholderTokenWithKey ( { input , regex : this . INDEXED_PLACEHOLDER_REGEX , parseKey : v => v . slice ( 1 ) } ) ; } getPlaceholderTokenWithKey ( { input , regex , parseKey } ) { const token = this . getTokenOnFirstMatch ( { input , regex , type : TokenTypes . PLACEHOLDER } ) ; if ( token ) { token . key = parseKey ( token . value ) ; } return token ; } getEscapedPlaceholderKey ( { key , quoteChar } ) { return key . replace ( new RegExp ( escapeRegExp ( '\\' + quoteChar ) , 'gu' ) , quoteChar ) ; } getNumberToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . NUMBER , regex : this . NUMBER_REGEX } ) ; } getOperatorToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . OPERATOR , regex : this . OPERATOR_REGEX } ) ; } getAmbiguosOperatorToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . OPERATOR , regex : this . AMBIGUOS_OPERATOR_REGEX } ) ; } getNoSpaceOperatorToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . NO_SPACE_OPERATOR , regex : this . NO_SPACE_OPERATOR_REGEX } ) ; } getReservedWordToken ( input , previousToken ) { if ( previousToken && previousToken . value && previousToken . value === '.' ) { return ; } return ( this . getToplevelReservedToken ( input ) || this . getNewlineReservedToken ( input ) || this . getTopLevelReservedTokenNoIndent ( input ) || this . getPlainReservedToken ( input ) ) ; } getToplevelReservedToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . RESERVED_TOP_LEVEL , regex : this . RESERVED_TOP_LEVEL_REGEX } ) ; } getNewlineReservedToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . RESERVED_NEWLINE , regex : this . RESERVED_NEWLINE_REGEX } ) ; } getPlainReservedToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . RESERVED , regex : this . RESERVED_PLAIN_REGEX } ) ; } getTopLevelReservedTokenNoIndent ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . RESERVED_TOP_LEVEL_NO_INDENT , regex : this . RESERVED_TOP_LEVEL_NO_INDENT_REGEX } ) ; } getWordToken ( input ) { return this . getTokenOnFirstMatch ( { input , type : TokenTypes . WORD , regex : this . WORD_REGEX } ) ; } getTokenOnFirstMatch ( { input , type , regex } ) { const matches = input . match ( regex ) ; if ( matches ) { return { type , value : matches [ 1 ] } ; } } }