export type Token = { symbol : string ; text : string ; } ; export class Symbol { static ofChar ( ch ) { return new Symbol ( ch , text => { return text === ch ; } ) ; } static ofChars ( ch ) { if ( ch . length === 0 ) { throw "empty symbol" ; } if ( ch [ 0 ] == null ) { throw "ch mustn't be null" ; } if ( ch . length === 0 ) { return this . ofChar ( ch [ 0 ] ) ; } return new Symbol ( ch [ 0 ] , text => { return ch . filter ( c => text === c ) . length > 0 ; } ) ; } private constructor ( public primary , private func ) { } isSymbol ( text ) { return this . func ( text ) ; } ; } export class Tokens { constructor ( private tokens ) { } public setTokenText ( symbol , text , keepSpace = false , create = false , separateSymbolAndText = false ) { let token = this . getToken ( symbol ) ; if ( token === null ) { if ( ! create ) { return null ; } if ( symbol instanceof Symbol ) { token = { symbol : symbol . primary , text } ; } else { token = { symbol , text } ; } if ( separateSymbolAndText && token . symbol !== '' && ! token . text . startsWith ( " " ) ) { token . text = ' ' + token . text ; } if ( this . tokens . length > 0 ) { const lastToken = this . tokens [ this . tokens . length - 1 ] ! ; if ( ! this . isTokenEndsWithSpace ( lastToken ) ) { lastToken . text += ' ' ; } } this . tokens . push ( token ) ; return token ; } this . replaceTokenText ( token , text , keepSpace ) ; return token ; } public length ( ) { return this . tokens . length ; } private replaceTokenText ( token , text , keepSpace = false ) { if ( ! keepSpace ) { token . text = text ; return ; } token . text = token . text . replace ( /^(\s*).*?(\s*)$/ , `$1${ text }$2` ) ; } private isTokenEndsWithSpace ( token ) { return token . text . match ( /^.*\s$/ ) ; } public getToken ( symbol ) { for ( let token of this . tokens ) { if ( symbol instanceof Symbol ) { if ( symbol . isSymbol ( token . symbol ) ) { return token ; } } else { if ( symbol === token . symbol ) { return token ; } } } return null ; } public getTokenText ( symbol , removeSpace = false ) { const token = this . getToken ( symbol ) ; if ( token === null ) { return null ; } if ( ! removeSpace ) { return token . text ; } return token . text . replace ( /^\s*(.*?)\s*$/ , `$1` ) ; } public removeToken ( symbol ) { this . tokens = this . tokens . filter ( token => ! symbol . isSymbol ( token . symbol ) ) ; } forEachTokens ( consumer ) { this . tokens . forEach ( consumer ) ; } public join ( ) { return this . tokens . map ( t => t . symbol + t . text ) . join ( "" ) ; } } export function splitBySymbol ( line , symbols ) { const chars = [ ... line ] ; let text = "" ; let currentToken = null ; const splitted = [ ] ; const fillPreviousToken = ( ) => { if ( currentToken === null ) { splitted . push ( { symbol : '' , text } ) ; } else { currentToken . text = text ; } } ; chars . forEach ( c => { let isSymbol = symbols . filter ( s => s . isSymbol ( c ) ) . length > 0 ; if ( isSymbol ) { fillPreviousToken ( ) ; currentToken = { symbol : c , text : '' } ; splitted . push ( currentToken ) ; text = '' ; } else { text += c ; } } ) ; if ( text . length > 0 ) { fillPreviousToken ( ) ; } return splitted ; }