/**
 * ## Reference
 * TeX source code:
 *  http import { splitGraphemes } from './grapheme-splitter' ; export type Token = string ; param s class Tokenizer { obeyspaces : boolean ; private readonly s : string | string [ ] ; private pos : number ; constructor ( s ) { this . s = splitGraphemes ( s ) ; this . pos = 0 ; this . obeyspaces = false ; } return end ( ) { return this . pos >= this . s . length ; } /**
     * Return the next char and advance
     */ get ( ) { return this . pos < this . s . length ? this . s [ this . pos ++ ] : '' ; } /**
     * Return the next char, but do not advance
     */ peek ( ) { return this . s [ this . pos ] ; } /**
     * Return the next substring matching regEx and advance.
     */ match ( regEx ) { const execResult = typeof this . s === 'string' ? regEx . exec ( this . s . slice ( this . pos ) ) : regEx . exec ( this . s . slice ( this . pos ) . join ( '' ) ) ; if ( execResult ?. [ 0 ] ) { this . pos += execResult [ 0 ] . length ; return execResult [ 0 ] ; } return '' ; } /**
     * Return the next token, or null.
     */ next ( ) { if ( this . end ( ) ) return null ; if ( ! this . obeyspaces && this . match ( /^[ \f\n\r\t\v\u00A0\u2028\u2029]+/ ) ) { return '<space>' ; } if ( this . obeyspaces && this . match ( /^[ \f\n\r\t\v\u00A0\u2028\u2029]/ ) ) { return '<space>' ; } const next = this . get ( ) ; if ( next === '\\' ) { if ( ! this . end ( ) ) { let command = this . match ( /^[a-zA-Z*]+/ ) ; if ( command ) { this . match ( /^[ \f\n\r\t\v\u00A0\u2028\u2029]*/ ) ; } else { command = this . get ( ) ; if ( command === ' ' ) { return '<space>' ; } } return '\\' + command ; } } else if ( next === '{' ) { return '<{>' ; } else if ( next === '}' ) { return '<}>' ; } else if ( next === '^' ) { if ( this . peek ( ) === '^' ) { this . get ( ) ; const hex = this . match ( /^(\^(\^(\^(\^[\da-f])?[\da-f])?[\da-f])?[\da-f])?[\da-f]{2}/ ) ; if ( hex ) { return String . fromCodePoint ( Number . parseInt ( hex . slice ( hex . lastIndexOf ( '^' ) + 1 ) , 16 ) ) ; } } return next ; } else if ( next === '#' ) { if ( ! this . end ( ) ) { let isParameter = false ; if ( /[\d?@]/ . test ( this . peek ( ) ) ) { isParameter = true ; if ( this . pos + 1 < this . s . length ) { const after = this . s [ this . pos + 1 ] ; isParameter = /[^\dA-Za-z]/ . test ( after ) ; } } if ( isParameter ) { return '#' + this . get ( ) ; } return '#' ; } } else if ( next === '$' ) { if ( this . peek ( ) === '$' ) { this . get ( ) ; return '<$$>' ; } return '<$>' ; } return next ; } } function expand ( lex , args ) { let result = [ ] ; let token = lex . next ( ) ; if ( token ) { if ( token === '\\relax' ) { } else if ( token === '\\noexpand' ) { token = lex . next ( ) ; if ( token ) { result . push ( token ) ; } } else if ( token === '\\obeyspaces' ) { lex . obeyspaces = true ; } else if ( token === '\\space' || token === '~' ) { result . push ( '<space>' ) ; } else if ( token === '\\bgroup' ) { result . push ( '<{>' ) ; } else if ( token === '\\egroup' ) { result . push ( '<}>' ) ; } else if ( token === '\\string' ) { token = lex . next ( ) ; if ( token ) { if ( token . startsWith ( '\\' ) ) { for ( const x of token ) { result . push ( x === '\\' ? '\\backslash' : x ) ; } } else if ( token === '<{>' ) { result . push ( '\\{' ) ; } else if ( token === '<space>' ) { result . push ( '~' ) ; } else if ( token === '<}>' ) { result . push ( '\\}' ) ; } } } else if ( token === '\\csname' ) { while ( lex . peek ( ) === '<space>' ) { lex . next ( ) ; } let command = '' ; let done = false ; let tokens = [ ] ; do { if ( tokens . length === 0 ) { if ( /^#[\d?@]$/ . test ( lex . peek ( ) ) ) { const parameter = lex . get ( ) . slice ( 1 ) ; tokens = tokenize ( args ?. ( parameter ) ?? args ?. ( '?' ) ?? '\\placeholder{}' , args ) ; token = tokens [ 0 ] ; } else { token = lex . next ( ) ; tokens = token ? [ token ] : [ ] ; } } done = tokens . length === 0 ; if ( ! done && token === '\\endcsname' ) { done = true ; tokens . shift ( ) ; } if ( ! done ) { done = token === '<$>' || token === '<$$>' || token === '<{>' || token === '<}>' || ( typeof token === 'string' && token . length > 1 && token . startsWith ( '\\' ) ) ; } if ( ! done ) { command += tokens . shift ( ) ; } } while ( ! done ) ; if ( command ) { result . push ( '\\' + command ) ; } result = result . concat ( tokens ) ; } else if ( token === '\\endcsname' ) { } else if ( token . length > 1 && token . startsWith ( '#' ) ) { const parameter = token . slice ( 1 ) ; result = result . concat ( tokenize ( args ?. ( parameter ) ?? args ?. ( '?' ) ?? '\\placeholder{}' , args ) ) ; } else { result . push ( token ) ; } } return result ; } param s export function tokenize ( s , args = null ) { const lines = s . toString ( ) . split ( /\r?\n/ ) ; let stream = '' ; let sep = '' ; for ( const line of lines ) { stream += sep ; sep = ' ' ; const m = line . match ( /((?:\\%)|[^%])*/ ) ; if ( m !== null ) stream += m [ 0 ] ; } const tokenizer = new Tokenizer ( stream ) ; let result = [ ] ; do { result = result . concat ( expand ( tokenizer , args ) ) ; } while ( ! tokenizer . end ( ) ) ; return result ; } export function joinLatex ( segments ) { let sep = '' ; let result = '' ; for ( const segment of segments ) { if ( segment ) { if ( /[a-zA-Z*]/ . test ( segment [ 0 ] ) ) { result += sep ; } sep = /\\[a-zA-Z]+\*?$/ . test ( segment ) ? ' ' : '' ; result += segment ; } } return result ; } export function tokensToString ( tokens ) { const result = joinLatex ( tokens . map ( ( token ) => { return ( { '<space>' : ' ' , '<$$>' : '$$' , '<$>' : '$' , '<{>' : '{' , '<}>' : '}' , } [ token ] ?? token ) ; } ) ) ; return result ; }