import { Locale } from '../types' ; import { getNormalizer } from './engines/normalizer' ; import { getStopwords } from './engines/stopwords' ; import { getTokenizer } from './engines/tokenizer' ; import { PreprocessEngines } from './types' ; export enum SEQUENCE_POSITION { PRE , POST } export class Preprocessor { engines : PreprocessEngines = { } ; constructor ( readonly locale , readonly maxLength ) { this . loadEngines ( ) ; } private loadEngines ( ) { try { this . engines . normalizer = getNormalizer ( this . locale ) ; this . engines . tokenizer = getTokenizer ( this . locale ) ; this . engines . stopwords = getStopwords ( this . locale ) ; } catch { console . warn ( `Engines not implemented for locale "${ this . locale }". Using default.` ) ; } } preprocess ( text , paddingValue ) { const normalizedText = this . normalize ( text ) ; const tokens = this . tokenize ( normalizedText ) ; const filteredTokens = this . removeStopwords ( tokens ) ; const stemmedTokens = this . stem ( filteredTokens ) ; const paddedSequence = this . pad ( stemmedTokens , paddingValue ) ; const truncatedSequence = this . truncate ( paddedSequence ) ; return truncatedSequence ; } normalize ( text ) { return this . engines . normalizer ? this . engines . normalizer . normalize ( text ) : text . toLowerCase ( ) ; } tokenize ( text ) { return this . engines . tokenizer ? this . engines . tokenizer . tokenize ( text ) : text . split ( ' ' ) ; } removeStopwords ( tokens ) { return this . engines . stopwords ? tokens . filter ( t => ! this . engines . stopwords . includes ( t ) ) : tokens ; } stem ( tokens ) { return this . engines . stemmer ? this . engines . stemmer . stem ( tokens ) : tokens ; } pad ( tokens , value , position = SEQUENCE_POSITION . POST ) { const difference = this . maxLength - tokens . length ; if ( difference > 0 ) { const padd = Array ( difference ) . fill ( value ) ; return position === SEQUENCE_POSITION . PRE ? padd . concat ( tokens ) : tokens . concat ( padd ) ; } return tokens ; } truncate ( tokens , position = SEQUENCE_POSITION . POST ) { const difference = this . maxLength - tokens . length ; if ( difference < 0 ) { return position === SEQUENCE_POSITION . PRE ? tokens . slice ( - this . maxLength ) : tokens . slice ( 0 , this . maxLength ) ; } return tokens ; } }