import { DATASEP , HYPHEN , NEW_LINE , SEPARATORS , SPACE , STRING_ENCLOSER , QUOTE , TILDE , BACKSLASH , HASH , AT , TRUE , TRUE_F , FALSE , FALSE_F , NULL , NULL_F } from './constants' ; import { Token } from '.' ; import ErrorCodes from '../errors/io-error-codes' ; import { InternetObjectSyntaxError } from '../errors/io-error' ; import { isUndefined } from '../utils/is' ; type NullableToken = Token | null ; const escapeCharMap = { '\\' : '\\' , '/' : '/' , '"' : '"' , b : '\b' , f : '\f' , n : '\n' , r : '\r' , t : '\t' } ; export default class Tokenizer { private _text : string ; private _tokens : Token [ ] = [ ] ; private _value = '' ; private _lastToken : NullableToken = null ; private _tokenLength = 0 ; private _done : boolean = false ; private _index : number = - 1 ; private _row : number = 1 ; private _col : number = 0 ; private _start : number = - 1 ; private _end : number = - 1 ; private _isQuoatedString : boolean = false ; private _isRawString : boolean = false ; private _isEscaping = false ; private _isCommenting = false ; public constructor ( text ) { this . _text = text ; } public read = ( ) : NullableToken => { if ( this . done ) return null ; const text = this . _text ; this . _lastToken = { } as Token ; this . _value = '' ; this . _start = - 1 ; this . _end = - 1 ; this . _tokenLength = 0 ; this . _isQuoatedString = false ; const token = this . _next ( ) ; if ( this . _done && token !== null && isUndefined ( token . col ) ) { return null ; } return token ; } ; public readAll = ( ) => { while ( ! this . _done ) { this . read ( ) ; } return this ; } ; public get done ( ) : boolean { return this . _done ; } public get length ( ) : number { return this . _tokens . length ; } public push = ( ... items ) : Tokenizer => { this . _tokens . push ( ... items ) ; return this ; } ; public get tokens ( ) { return this . _tokens ; } public get ( index ) { return this . _tokens [ index ] ; } private _returnToken = ( ) : NullableToken => { const token = this . _lastToken ; if ( token === null ) return null ; let type = 'string' ; let value = this . _value ; token . token = this . _text . substring ( this . _start , this . _end + ( this . _isQuoatedString ? 1 : 1 ) ) ; if ( token . token . startsWith ( STRING_ENCLOSER ) ) { if ( ! token . token . endsWith ( STRING_ENCLOSER ) ) { throw new InternetObjectSyntaxError ( ErrorCodes . stringNotClosed , 'String not closed, expecting " at the end of the string.' , token ) ; } } else if ( token . token . startsWith ( QUOTE ) ) { if ( ! token . token . endsWith ( QUOTE ) ) { throw new InternetObjectSyntaxError ( ErrorCodes . stringNotClosed , "String not closed, expecting ' at the end of the string." , token ) ; } } else if ( token . token . startsWith ( HASH ) ) { type = 'comment' ; value = value . substr ( 1 ) . trim ( ) ; } else { value = value . trim ( ) ; let numVal = Number ( value ) ; if ( SEPARATORS . indexOf ( value ) >= 0 || value === TILDE ) { type = 'sep' ; } else if ( value === DATASEP ) { type = 'datasep' ; } else if ( ! isNaN ( numVal ) && value . trim ( ) !== '' ) { value = numVal ; type = 'number' ; } else if ( value === TRUE || value === TRUE_F ) { value = true ; type = 'boolean' ; } else if ( value === FALSE || value === FALSE_F ) { value = false ; type = 'boolean' ; } else if ( value === NULL || value === NULL_F ) { value = null ; type = 'null' ; } } token . value = value ; token . type = type ; this . _tokens . push ( token ) ; this . skipNextWhiteSpaces ( ) ; return token ; } ; private _next = ( ) : NullableToken => { this . _col += 1 ; const index = ++ this . _index ; let ch = this . _text [ index ] ; let nextCh = this . _text [ index + 1 ] ; let isWS = ch <= SPACE ; let isNewLine = ch === NEW_LINE ; let isNextNewLine = nextCh === NEW_LINE ; let isSep = SEPARATORS . indexOf ( ch ) >= 0 ; let isNextSep = SEPARATORS . indexOf ( nextCh ) >= 0 ; const isCollectionSep = ch === TILDE ; let isNextCollectionSep = nextCh === TILDE ; const isStarted = this . _start !== - 1 ; const isDataSep = this . _text . substr ( index - 2 , 3 ) === DATASEP ; let isNextDataSep = this . _text . substr ( index + 1 , 3 ) === DATASEP ; if ( isUndefined ( ch ) ) { if ( this . _isEscaping ) { throw new InternetObjectSyntaxError ( ErrorCodes . incompleteEscapeSequence , 'End of the text reached before finishing the escape sequence.' ) ; } this . _done = true ; return this . _returnToken ( ) ; } if ( this . _isCommenting ) { this . _end = index ; this . _value += ch ; if ( isNextNewLine ) { this . _isCommenting = false ; return this . _returnToken ( ) ; } this . _tokenLength += 1 ; return this . _next ( ) ; } const token = this . _lastToken ; if ( ! token ) return null ; if ( isWS ) { if ( ch === '\r' ) { if ( nextCh === '\n' ) { return this . _next ( ) ; } ch = '\n' ; isNewLine = true ; } if ( this . _tokenLength > 0 ) this . _value += ch ; if ( isNewLine ) { this . _row += 1 ; this . _col = 0 ; } if ( ! this . _isQuoatedString && ! this . _isRawString && isStarted ) { if ( isNextSep || isNextCollectionSep || isNextDataSep ) { if ( this . _col === 0 ) this . _col = 1 ; return this . _returnToken ( ) ; } else if ( nextCh === HASH ) { return this . _returnToken ( ) ; } } return this . _next ( ) ; } this . _end = index ; if ( this . _start === - 1 ) { this . _start = index ; if ( ! token . col ) { token . index = this . _start ; token . col = this . _col ; token . row = this . _row ; } } this . _tokenLength += 1 ; if ( this . _tokenLength === 1 ) { if ( ch === QUOTE ) { this . _isRawString = true ; return this . _next ( ) ; } else if ( ch === STRING_ENCLOSER ) { this . _isQuoatedString = true ; return this . _next ( ) ; } } if ( this . _isQuoatedString ) { if ( ch === STRING_ENCLOSER ) { if ( this . _isEscaping === false ) { this . _isQuoatedString = false ; return this . _returnToken ( ) ; } } if ( ch === BACKSLASH && this . _isEscaping === false ) { this . _isEscaping = true ; return this . _next ( ) ; } if ( this . _isEscaping ) { this . _isEscaping = false ; this . _value += ch in escapeCharMap ? escapeCharMap [ ch ] : ch ; return this . _next ( ) ; } } if ( this . _isRawString ) { if ( ch === QUOTE ) { if ( this . _isEscaping === false && nextCh === QUOTE ) { this . _isEscaping = true ; return this . _next ( ) ; } if ( this . _isEscaping ) { this . _isEscaping = false ; this . _value += "'" ; return this . _next ( ) ; } if ( this . _isEscaping === false ) { this . _isRawString = false ; return this . _returnToken ( ) ; } } this . _value += ch ; return this . _next ( ) ; } if ( ! this . _isQuoatedString && ! this . _isRawString ) { if ( nextCh === HASH ) { this . _isCommenting = true ; return this . _returnToken ( ) ; } if ( ch === HASH ) { this . _isCommenting = true ; this . _value += ch ; return this . _next ( ) ; } if ( isSep || isNextSep || isCollectionSep || isNextCollectionSep ) { this . _value += ch ; return this . _returnToken ( ) ; } else if ( ch === HYPHEN ) { if ( isDataSep ) { this . _value = '---' ; return this . _returnToken ( ) ; } } } this . _value += ch ; return this . _next ( ) ; } ; private skipNextWhiteSpaces ( ) { const ch = this . _text [ this . _index + 1 ] ; if ( isUndefined ( ch ) ) { this . _done = true ; return ; } if ( ch > SPACE ) return ; this . _col += 1 ; ++ this . _index ; if ( this . _tokenLength > 0 ) this . _value += ch ; if ( ch === NEW_LINE ) { this . _row += 1 ; this . _col = 0 ; } this . skipNextWhiteSpaces ( ) ; } private get _node ( ) { return { index : this . _index , row : this . _row , col : this . _col } ; } }