import moo from 'moo' ; import { regEx } from '../../../util/regex' ; import { TokenType } from './common' ; import type { StringInterpolation , Token } from './types' ; const escapedCharRegex = /\\['"bfnrt\\]/ ; const escapedChars = { [ TokenType . EscapedChar ] : { match : escapedCharRegex , value : ( x ) : string => ( { "\\'" : "'" , '\\"' : '"' , '\\b' : '\b' , '\\f' : '\f' , '\\n' : '\n' , '\\r' : '\r' , '\\t' : '\t' , '\\\\' : '\\' , } [ x ] ?? x ) , } , } ; const lexer = moo . states ( { main : { [ TokenType . LineComment ] : { match : /\/\/.*?$/ } , [ TokenType . MultiComment ] : { match : /\/\*[^]*?\*\// , lineBreaks : true } , [ TokenType . Newline ] : { match : /\r?\n/ , lineBreaks : true } , [ TokenType . Space ] : { match : /[ \t\r]+/ } , [ TokenType . Semicolon ] : ';' , [ TokenType . Colon ] : ':' , [ TokenType . Dot ] : '.' , [ TokenType . Comma ] : ',' , [ TokenType . Operator ] : /(?:==|\+=?|-=?|\/=?|\*\*?|\.+|:)/ , [ TokenType . Assignment ] : '=' , [ TokenType . Word ] : { match : /[a-zA-Z$_][a-zA-Z0-9$_]*/ } , [ TokenType . LeftParen ] : { match : '(' } , [ TokenType . RightParen ] : { match : ')' } , [ TokenType . LeftBracket ] : { match : '[' } , [ TokenType . RightBracket ] : { match : ']' } , [ TokenType . LeftBrace ] : { match : '{' , push : 'main' } , [ TokenType . RightBrace ] : { match : '}' , pop : 1 } , [ TokenType . TripleSingleQuotedStart ] : { match : "'''" , push : TokenType . TripleSingleQuotedStart , } , [ TokenType . TripleDoubleQuotedStart ] : { match : '"""' , push : TokenType . TripleDoubleQuotedStart , } , [ TokenType . SingleQuotedStart ] : { match : "'" , push : TokenType . SingleQuotedStart , } , [ TokenType . DoubleQuotedStart ] : { match : '"' , push : TokenType . DoubleQuotedStart , } , [ TokenType . UnknownFragment ] : moo . fallback , } , [ TokenType . TripleSingleQuotedStart ] : { ... escapedChars , [ TokenType . TripleQuotedFinish ] : { match : "'''" , pop : 1 } , [ TokenType . Chars ] : moo . fallback , } , [ TokenType . TripleDoubleQuotedStart ] : { ... escapedChars , [ TokenType . TripleQuotedFinish ] : { match : '"""' , pop : 1 } , [ TokenType . Chars ] : moo . fallback , } , [ TokenType . SingleQuotedStart ] : { ... escapedChars , [ TokenType . SingleQuotedFinish ] : { match : "'" , pop : 1 } , [ TokenType . Chars ] : moo . fallback , } , [ TokenType . DoubleQuotedStart ] : { ... escapedChars , [ TokenType . DoubleQuotedFinish ] : { match : '"' , pop : 1 } , variable : { match : /\${\s*[a-zA-Z_][a-zA-Z0-9_]*(?:\s*\.\s*[a-zA-Z_][a-zA-Z0-9_]*)*\s*}|\$[a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*/ , value : ( x ) : string => x . replace ( regEx ( /^\${?\s*/ ) , '' ) . replace ( regEx ( /\s*}$/ ) , '' ) , } , [ TokenType . IgnoredInterpolationStart ] : { match : /\${/ , push : TokenType . IgnoredInterpolationStart , } , [ TokenType . Chars ] : moo . fallback , } , [ TokenType . IgnoredInterpolationStart ] : { [ TokenType . LeftBrace ] : { match : '{' , push : TokenType . IgnoredInterpolationStart , } , [ TokenType . RightBrace ] : { match : '}' , pop : 1 } , [ TokenType . UnknownFragment ] : moo . fallback , } , } ) ; function processChars ( acc , token ) { const tokenType = token . type ; const prevToken = acc [ acc . length - 1 ] ; if ( [ TokenType . Chars , TokenType . EscapedChar ] . includes ( tokenType ) ) { if ( prevToken ?. type === TokenType . String ) { prevToken . value += token . value ; } else { acc . push ( { ... token , type : TokenType . String } ) ; } } else { acc . push ( token ) ; } return acc ; } export function isInterpolationToken ( token ) { return token ?. type === TokenType . StringInterpolation ; } function processInterpolation ( acc , token ) { if ( token . type === TokenType . DoubleQuotedStart ) { const interpolationToken = { type : TokenType . StringInterpolation , children : [ ] , isValid : true , isComplete : false , offset : token . offset + 1 , value : '' , } ; acc . push ( interpolationToken ) ; return acc ; } const prevToken = acc [ acc . length - 1 ] ; if ( isInterpolationToken ( prevToken ) && ! prevToken . isComplete ) { const type = token . type ; if ( type === TokenType . DoubleQuotedFinish ) { if ( prevToken . isValid && prevToken . children . every ( ( { type : t } ) => t === TokenType . String ) ) { acc [ acc . length - 1 ] = { type : TokenType . String , value : prevToken . children . map ( ( { value } ) => value ) . join ( '' ) , offset : prevToken . offset , } ; return acc ; } prevToken . isComplete = true ; } else if ( type === TokenType . String || type === TokenType . Variable ) { prevToken . children . push ( token ) ; } else { prevToken . children . push ( token ) ; prevToken . isValid = false ; } } else { acc . push ( token ) ; } return acc ; } const filteredTokens = [ TokenType . Space , TokenType . LineComment , TokenType . MultiComment , TokenType . Newline , TokenType . Semicolon , TokenType . SingleQuotedStart , TokenType . SingleQuotedFinish , TokenType . DoubleQuotedFinish , TokenType . TripleSingleQuotedStart , TokenType . TripleDoubleQuotedStart , TokenType . TripleQuotedFinish , ] ; function filterTokens ( { type } ) { return ! filteredTokens . includes ( type ) ; } export function extractRawTokens ( input ) { lexer . reset ( input ) ; return Array . from ( lexer ) . map ( ( { type , offset , value } ) => ( { type , offset , value } as Token ) ) ; } export function processTokens ( tokens ) { return tokens . reduce ( processChars , [ ] ) . reduce ( processInterpolation , [ ] ) . filter ( filterTokens ) ; } export function tokenize ( input ) { return processTokens ( extractRawTokens ( input ) ) ; }