import { IAidaTokenizer } from '../../types' ; class EnglishTokenizer implements IAidaTokenizer { public FILTER_CHARS_REGEXP = /[^a-z0-9\.,\?\'"!@#\$%\^&\*\(\)-_=\+;:<>\/\\\|\}\{\[\]`~ ]/gi ; public WORD_SEPARATORS_REGEXP = /([\ \.\,\%\*\-\=\+\;\|\`\~])/g ; public NON_ALPHANUMERIC_REGEXP = /[^a-z0-9]/gi ; public UNKNOWN_NGRAM_KEY = '__' ; public NUMBERS_MAP : { [ key ] : string ; } = { zero : '0' , one : '1' , two : '2' , three : '3' , four : '4' , five : '5' , six : '6' , seven : '7' , eight : '8' , nine : '9' } ; public sanitizeSentence = ( sentence ) : string => { return sentence . trim ( ) . toLowerCase ( ) . replace ( this . FILTER_CHARS_REGEXP , '' ) ; } ; public splitSentenceToWords = ( sentence ) : string [ ] => { return this . sanitizeSentence ( sentence ) . split ( this . WORD_SEPARATORS_REGEXP ) . map ( w => w . trim ( ) ) . filter ( w => ! ! w ) ; } ; public splitWordToBiGrams = ( word ) : string [ ] => { const ngram = 2 ; const grams = [ ] ; let index = word . length - ngram + 1 ; if ( index < 1 ) { return grams ; } while ( index -- ) { grams [ index ] = word . substr ( index , ngram ) ; } return grams ; } ; public joinWordsToSentence = ( words ) : string => words . join ( ' ' ) ; } export default new EnglishTokenizer ( ) ;