import * as Iterator from "../../util/Iterator" ; /**
 * The different types of tokens that can be generated by a basic Tokenizer.
 * This tokenizer knows nothing about JSON, it just recognizes basic punctuation
 * and groups of digits and letters (not grouped into numbers or strings etc).
 */ export const enum TokenType { LeftCurlyBracket , RightCurlyBracket , LeftSquareBracket , RightSquareBracket , LeftParenthesis , RightParenthesis , Underscore , Period , Dash , Plus , Comma , Colon , SingleQuote , DoubleQuote , Backslash , ForwardSlash , Asterisk , Space , Tab , NewLine , CarriageReturn , CarriageReturnNewLine , Letters , Digits , Unrecognized } /**
 * An individual Token that has been parsed by a basic Tokenizer.
 */ export class Token { constructor ( private _text , private _type ) { } /**
     * Gets the original string that this basic token was parsed from.
     */ public toString ( ) { return this . _text ; } /**
     * Get the number of characters that make up this token.
     */ public length ( ) { return this . _text . length ; } /**
     * Get the type of this token.
     */ public getType ( ) { return this . _type ; } /**
     * Convenient way of seeing what this token represents in the debugger, shouldn't be used for production code
     */ public get __debugDisplay ( ) : string { return `<${ this . toString ( ) }>` ; } } export const LeftCurlyBracket = new Token ( "{" , TokenType . LeftCurlyBracket ) ; export const RightCurlyBracket = new Token ( "}" , TokenType . RightCurlyBracket ) ; export const LeftSquareBracket = new Token ( "[" , TokenType . LeftSquareBracket ) ; export const RightSquareBracket = new Token ( "]" , TokenType . RightSquareBracket ) ; export const LeftParenthesis = new Token ( "(" , TokenType . LeftParenthesis ) ; export const RightParenthesis = new Token ( ")" , TokenType . RightParenthesis ) ; export const Underscore = new Token ( "_" , TokenType . Underscore ) ; export const Period = new Token ( "." , TokenType . Period ) ; export const Dash = new Token ( "-" , TokenType . Dash ) ; export const Plus = new Token ( "+" , TokenType . Plus ) ; export const Comma = new Token ( "," , TokenType . Comma ) ; export const Colon = new Token ( ":" , TokenType . Colon ) ; export const SingleQuote = new Token ( `'` , TokenType . SingleQuote ) ; export const DoubleQuote = new Token ( `"` , TokenType . DoubleQuote ) ; export const Backslash = new Token ( "\\" , TokenType . Backslash ) ; export const ForwardSlash = new Token ( "/" , TokenType . ForwardSlash ) ; export const Asterisk = new Token ( "*" , TokenType . Asterisk ) ; export const Space = new Token ( " " , TokenType . Space ) ; export const Tab = new Token ( "\t" , TokenType . Tab ) ; export const NewLine = new Token ( "\n" , TokenType . NewLine ) ; export const CarriageReturn = new Token ( "\r" , TokenType . CarriageReturn ) ; export const CarriageReturnNewLine = new Token ( "\r\n" , TokenType . CarriageReturnNewLine ) ; /**
 * Create a Letters Token from the provided text.
 */ export function Letters ( text ) { return new Token ( text , TokenType . Letters ) ; } /**
 * Create a Digits Token from the provided text.
 */ export function Digits ( text ) { return new Token ( text , TokenType . Digits ) ; } /**
 * Create an Unrecognized Token from the provided text.
 */ export function Unrecognized ( text ) { return new Token ( text , TokenType . Unrecognized ) ; } /**
 * A Tokenizer that breaks a provided text string into a sequence of basic Tokens.
 */ export class Tokenizer implements Iterator . Iterator < Token > { private _textLength : number ; private _textIndex : number = - 1 ; private _currentToken : Token | undefined ; constructor ( private _text ) { this . _textLength = _text ? _text . length : 0 ; } /**
     * Convenient way of seeing what this object represents in the debugger, shouldn't be used for production code
     */ public get __debugDisplay ( ) : string { const charactersBeforeCurrent = 25 ; const charactersAfterCurrent = 50 ; return `${ this . _text . slice ( this . _textIndex - charactersBeforeCurrent , this . _textIndex ) }<<${ this . _currentToken ? this . _currentToken . toString ( ) : '' }>>${ this . _text . slice ( this . _textIndex + 1 , this . _textIndex + 1 + charactersAfterCurrent ) }` ; } /**
     * Get whether this Tokenizer has started tokenizing its text string.
     */ public hasStarted ( ) { return 0 <= this . _textIndex ; } /**
     * Get the current Token that this Tokenizer has parsed from the source text. If this Tokenizer
     * hasn't started parsing the source text, or if it has parsed the entire source text, then this
     * will return undefined.
     */ public current ( ) { return this . _currentToken ; } /**
     * Get the current character that this Tokenizer is pointing at.
     */ private get currentCharacter ( ) : string | undefined { return 0 <= this . _textIndex && this . _textIndex < this . _textLength ? this . _text [ this . _textIndex ] : undefined ; } /**
     * Move this Tokenizer to the next character in its source text.
     */ private nextCharacter ( ) { ++ this . _textIndex ; } returns public moveNext ( ) { if ( ! this . hasStarted ( ) ) { this . nextCharacter ( ) ; } if ( ! this . currentCharacter ) { this . _currentToken = undefined ; } else { switch ( this . currentCharacter ) { case "{" : this . _currentToken = LeftCurlyBracket ; this . nextCharacter ( ) ; break ; case "}" : this . _currentToken = RightCurlyBracket ; this . nextCharacter ( ) ; break ; case "[" : this . _currentToken = LeftSquareBracket ; this . nextCharacter ( ) ; break ; case "]" : this . _currentToken = RightSquareBracket ; this . nextCharacter ( ) ; break ; case "(" : this . _currentToken = LeftParenthesis ; this . nextCharacter ( ) ; break ; case ")" : this . _currentToken = RightParenthesis ; this . nextCharacter ( ) ; break ; case "_" : this . _currentToken = Underscore ; this . nextCharacter ( ) ; break ; case "." : this . _currentToken = Period ; this . nextCharacter ( ) ; break ; case "-" : this . _currentToken = Dash ; this . nextCharacter ( ) ; break ; case "+" : this . _currentToken = Plus ; this . nextCharacter ( ) ; break ; case "," : this . _currentToken = Comma ; this . nextCharacter ( ) ; break ; case ":" : this . _currentToken = Colon ; this . nextCharacter ( ) ; break ; case `'` : this . _currentToken = SingleQuote ; this . nextCharacter ( ) ; break ; case `"` : this . _currentToken = DoubleQuote ; this . nextCharacter ( ) ; break ; case "\\" : this . _currentToken = Backslash ; this . nextCharacter ( ) ; break ; case "/" : this . _currentToken = ForwardSlash ; this . nextCharacter ( ) ; break ; case "*" : this . _currentToken = Asterisk ; this . nextCharacter ( ) ; break ; case "\n" : this . _currentToken = NewLine ; this . nextCharacter ( ) ; break ; case "\r" : this . nextCharacter ( ) ; if ( this . currentCharacter && this . currentCharacter . toString ( ) === "\n" ) { this . _currentToken = CarriageReturnNewLine ; this . nextCharacter ( ) ; } else { this . _currentToken = CarriageReturn ; } break ; case " " : this . _currentToken = Space ; this . nextCharacter ( ) ; break ; case "\t" : this . _currentToken = Tab ; this . nextCharacter ( ) ; break ; default : if ( isLetter ( this . currentCharacter ) ) { this . _currentToken = Letters ( this . readWhile ( isLetter ) ) ; } else if ( isDigit ( this . currentCharacter ) ) { this . _currentToken = Digits ( this . readWhile ( isDigit ) ) ; } else { this . _currentToken = Unrecognized ( this . currentCharacter ) ; this . nextCharacter ( ) ; } break ; } } return ! ! this . _currentToken ; } /**
     * Read and return a sequence of characters from the source text that match the provided
     * condition function.
     */ private readWhile ( condition ) { let result = this . currentCharacter || '' ; this . nextCharacter ( ) ; while ( this . currentCharacter && condition ( this . currentCharacter ) ) { result += this . currentCharacter ; this . nextCharacter ( ) ; } return result ; } } /**
 * Get whether the provided character is a letter or not.
 */ function isLetter ( character ) { return ( "a" <= character && character <= "z" ) || ( "A" <= character && character <= "Z" ) ; } /**
 * Get whether the provided character is a digit or not.
 */ function isDigit ( character ) { return "0" <= character && character <= "9" ; }