license import * as chars from "../ast/chars" ; export enum TokenType { Character , Identifier , Keyword , String , Operator , Number , Error } const KEYWORDS = [ "var" , "let" , "as" , "null" , "undefined" , "true" , "false" , "if" , "else" , "this" ] ; export class Lexer { tokenize ( text ) { const scanner = new Scanner ( text ) ; const tokens = [ ] ; let token = scanner . scanToken ( ) ; while ( token != null ) { tokens . push ( token ) ; token = scanner . scanToken ( ) ; } return tokens ; } } export class Token { constructor ( public index , public type , public numValue , public strValue ) { } isCharacter ( code ) { return this . type === TokenType . Character && this . numValue === code ; } isNumber ( ) { return this . type === TokenType . Number ; } isString ( ) { return this . type === TokenType . String ; } isOperator ( operater ) { return this . type === TokenType . Operator && this . strValue === operater ; } isIdentifier ( ) { return this . type === TokenType . Identifier ; } isKeyword ( ) { return this . type === TokenType . Keyword ; } isKeywordLet ( ) { return this . type === TokenType . Keyword && this . strValue === "let" ; } isKeywordAs ( ) { return this . type === TokenType . Keyword && this . strValue === "as" ; } isKeywordNull ( ) { return this . type === TokenType . Keyword && this . strValue === "null" ; } isKeywordUndefined ( ) { return this . type === TokenType . Keyword && this . strValue === "undefined" ; } isKeywordTrue ( ) { return this . type === TokenType . Keyword && this . strValue === "true" ; } isKeywordFalse ( ) { return this . type === TokenType . Keyword && this . strValue === "false" ; } isKeywordThis ( ) { return this . type === TokenType . Keyword && this . strValue === "this" ; } isError ( ) { return this . type === TokenType . Error ; } toNumber ( ) { return this . type === TokenType . Number ? this . numValue : - 1 ; } toString ( ) { switch ( this . type ) { case TokenType . Character : case TokenType . Identifier : case TokenType . Keyword : case TokenType . Operator : case TokenType . String : case TokenType . Error : return this . strValue ; case TokenType . Number : return this . numValue . toString ( ) ; default : return null ; } } } function newCharacterToken ( index , code ) { return new Token ( index , TokenType . Character , code , String . fromCharCode ( code ) ) ; } function newIdentifierToken ( index , text ) { return new Token ( index , TokenType . Identifier , 0 , text ) ; } function newKeywordToken ( index , text ) { return new Token ( index , TokenType . Keyword , 0 , text ) ; } function newOperatorToken ( index , text ) { return new Token ( index , TokenType . Operator , 0 , text ) ; } function newStringToken ( index , text ) { return new Token ( index , TokenType . String , 0 , text ) ; } function newNumberToken ( index , n ) { return new Token ( index , TokenType . Number , n , "" ) ; } function newErrorToken ( index , message ) { return new Token ( index , TokenType . Error , 0 , message ) ; } export const EOF = new Token ( - 1 , TokenType . Character , 0 , "" ) ; class Scanner { length : number ; peek = 0 ; index = - 1 ; constructor ( public input ) { this . length = input . length ; this . advance ( ) ; } advance ( ) { this . peek = ++ this . index >= this . length ? chars . $EOF : this . input . charCodeAt ( this . index ) ; } scanToken ( ) { const input = this . input ; const length = this . length ; let peek = this . peek ; let index = this . index ; while ( peek <= chars . $SPACE ) { if ( ++ index >= length ) { peek = chars . $EOF ; break ; } else { peek = input . charCodeAt ( index ) ; } } this . peek = peek ; this . index = index ; if ( index >= length ) { return null ; } if ( isIdentifierStart ( peek ) ) { return this . scanIdentifier ( ) ; } if ( chars . isDigit ( peek ) ) { return this . scanNumber ( index ) ; } const start = index ; switch ( peek ) { case chars . $PERIOD : this . advance ( ) ; return chars . isDigit ( this . peek ) ? this . scanNumber ( start ) : newCharacterToken ( start , chars . $PERIOD ) ; case chars . $LPAREN : case chars . $RPAREN : case chars . $LBRACE : case chars . $RBRACE : case chars . $LBRACKET : case chars . $RBRACKET : case chars . $COMMA : case chars . $COLON : case chars . $SEMICOLON : return this . scanCharacter ( start , peek ) ; case chars . $SQ : case chars . $DQ : return this . scanString ( ) ; case chars . $HASH : case chars . $PLUS : case chars . $MINUS : case chars . $STAR : case chars . $SLASH : case chars . $PERCENT : case chars . $CARET : return this . scanOperator ( start , String . fromCharCode ( peek ) ) ; case chars . $QUESTION : return this . scanComplexOperator ( start , "?" , chars . $PERIOD , "." ) ; case chars . $LT : case chars . $GT : return this . scanComplexOperator ( start , String . fromCharCode ( peek ) , chars . $EQ , "=" ) ; case chars . $BANG : case chars . $EQ : return this . scanComplexOperator ( start , String . fromCharCode ( peek ) , chars . $EQ , "=" , chars . $EQ , "=" ) ; case chars . $AMPERSAND : return this . scanComplexOperator ( start , "&" , chars . $AMPERSAND , "&" ) ; case chars . $BAR : return this . scanComplexOperator ( start , "|" , chars . $BAR , "|" ) ; case chars . $NBSP : while ( chars . isWhitespace ( this . peek ) ) { this . advance ( ) ; } return this . scanToken ( ) ; } this . advance ( ) ; return this . error ( `Unexpected character [${ String . fromCharCode ( peek ) }]` , 0 ) ; } scanCharacter ( start , code ) { this . advance ( ) ; return newCharacterToken ( start , code ) ; } scanOperator ( start , str ) { this . advance ( ) ; return newOperatorToken ( start , str ) ; } param start param one param twoCode param two param threeCode param three scanComplexOperator ( start , one , twoCode , two , threeCode ? , three ? ) { this . advance ( ) ; let str = one ; if ( this . peek === twoCode ) { this . advance ( ) ; str += two ; } if ( threeCode != null && this . peek === threeCode ) { this . advance ( ) ; str += three ; } return newOperatorToken ( start , str ) ; } scanIdentifier ( ) { const start = this . index ; this . advance ( ) ; while ( isIdentifierPart ( this . peek ) ) { this . advance ( ) ; } const str = this . input . substring ( start , this . index ) ; return KEYWORDS . indexOf ( str ) > - 1 ? newKeywordToken ( start , str ) : newIdentifierToken ( start , str ) ; } scanNumber ( start ) { let simple = this . index === start ; this . advance ( ) ; while ( true ) { if ( chars . isDigit ( this . peek ) ) { } else if ( this . peek === chars . $PERIOD ) { simple = false ; } else if ( isExponentStart ( this . peek ) ) { this . advance ( ) ; if ( isExponentSign ( this . peek ) ) { this . advance ( ) ; } if ( ! chars . isDigit ( this . peek ) ) { return this . error ( "Invalid exponent" , - 1 ) ; } simple = false ; } else { break ; } this . advance ( ) ; } const str = this . input . substring ( start , this . index ) ; const value = simple ? parseIntAutoRadix ( str ) : parseFloat ( str ) ; return newNumberToken ( start , value ) ; } scanString ( ) { const start = this . index ; const quote = this . peek ; this . advance ( ) ; let buffer = "" ; let marker = this . index ; const input = this . input ; while ( this . peek !== quote ) { if ( this . peek === chars . $BACKSLASH ) { buffer += input . substring ( marker , this . index ) ; this . advance ( ) ; let unescapedCode ; this . peek = this . peek ; if ( this . peek === chars . $u ) { const hex = input . substring ( this . index + 1 , this . index + 5 ) ; if ( /^[0-9a-f]+$/i . test ( hex ) ) { unescapedCode = parseInt ( hex , 16 ) ; } else { return this . error ( `Invalid unicode escape [\\u${ hex }]` , 0 ) ; } for ( let i = 0 ; i < 5 ; i ++ ) { this . advance ( ) ; } } else { unescapedCode = unescape ( this . peek ) ; this . advance ( ) ; } buffer += String . fromCharCode ( unescapedCode ) ; marker = this . index ; } else if ( this . peek === chars . $EOF ) { return this . error ( "Unterminated quote" , 0 ) ; } else { this . advance ( ) ; } } const last = input . substring ( marker , this . index ) ; this . advance ( ) ; return newStringToken ( start , buffer + last ) ; } error ( message , offset ) { const position = this . index + offset ; return newErrorToken ( position , `Lexer Error: ${ message } at column ${ position } in expression [${ this . input }]` ) ; } } function isIdentifierStart ( code ) { return ( ( chars . $a <= code && code <= chars . $z ) || ( chars . $A <= code && code <= chars . $Z ) || code === chars . $_ || code === chars . $$ ) ; } export function isIdentifier ( input ) { if ( input . length === 0 ) { return false ; } const scanner = new Scanner ( input ) ; if ( ! isIdentifierStart ( scanner . peek ) ) { return false ; } scanner . advance ( ) ; while ( scanner . peek !== chars . $EOF ) { if ( ! isIdentifierPart ( scanner . peek ) ) { return false ; } scanner . advance ( ) ; } return true ; } function isIdentifierPart ( code ) { return chars . isAsciiLetter ( code ) || chars . isDigit ( code ) || code === chars . $_ || code === chars . $$ ; } function isExponentStart ( code ) { return code === chars . $e || code === chars . $E ; } function isExponentSign ( code ) { return code === chars . $MINUS || code === chars . $PLUS ; } export function isQuote ( code ) { return code === chars . $SQ || code === chars . $DQ || code === chars . $BT ; } function unescape ( code ) { switch ( code ) { case chars . $n : return chars . $LF ; case chars . $f : return chars . $FF ; case chars . $r : return chars . $CR ; case chars . $t : return chars . $TAB ; case chars . $v : return chars . $VTAB ; default : return code ; } } function parseIntAutoRadix ( text ) { const result = parseInt ( text , 10 ) ; if ( isNaN ( result ) ) { throw new Error ( "Invalid integer literal when parsing " + text ) ; } return result ; }