import * as fs from 'fs' ; import * as path from 'path' ; import * as readline from 'readline' ; import englishTokenizer from '../languages/en/EnglishTokenizer' ; import spanishTokenizer from '../languages/es/SpanishTokenizer' ; function getTokenizerAndVectors ( language ) { const lang = language ? language . toLowerCase ( ) : language ; if ( lang === 'en' ) { return { fastTextVectorsFile : path . resolve ( __dirname , '../languages/en/fastText.en.vec' ) , outputDictionary : path . resolve ( __dirname , '../languages/en/dict.json' ) , outputDictionaryExtension : path . resolve ( __dirname , '../languages/en/dict.trigrams.json' ) , tokenizer : englishTokenizer } ; } else if ( lang === 'es' ) { return { fastTextVectorsFile : path . resolve ( __dirname , '../languages/es/fastText.es.vec' ) , outputDictionary : path . resolve ( __dirname , '../languages/es/dict.json' ) , outputDictionaryExtension : path . resolve ( __dirname , '../languages/es/dict.trigrams.json' ) , tokenizer : spanishTokenizer } ; } throw new Error ( "Invalid config language. Only 'en' and 'es' are supported." ) ; } const createDictionaryFromFastText = ( language ) => { if ( ! language ) { throw new Error ( "Invalid Config. 'config.embeddings.fastTextFile' must be provided to generate dictionary." ) ; } const { fastTextVectorsFile , tokenizer , outputDictionary , outputDictionaryExtension } = getTokenizerAndVectors ( language ) ; console . log ( 'Generating dictioanry with this language options: ' , JSON . stringify ( { fastTextVectorsFile , outputDictionary , outputDictionaryExtension } , null , 2 ) ) ; const lineReader = readline . createInterface ( { input : fs . createReadStream ( fastTextVectorsFile ) } ) ; const dictionary = new Map ( ) as Map < string , number [ ] > ; const dictionaryExtension = new Map ( ) as Map < string , number [ ] > ; lineReader . on ( 'line' , ( line ) => { const spaceSepValues = line . trim ( ) . split ( ' ' ) ; const values = spaceSepValues . slice ( 1 ) . map ( n => parseFloat ( n ) ) ; const keyword = spaceSepValues [ 0 ] . toLowerCase ( ) ; const containsNonAsciiCharacters = keyword . match ( tokenizer . FILTER_CHARS_REGEXP ) ? true : false ; if ( containsNonAsciiCharacters ) { return ; } if ( spaceSepValues . length === 2 ) { dictionary . set ( tokenizer . UNKNOWN_NGRAM_KEY , new Array ( 300 ) . fill ( 0 ) ) ; return console . log ( 'Words / Dimensions: ' , spaceSepValues . join ( ' / ' ) , 'Please wait, this may take a while...' ) ; } const nonAlphanumericChars = keyword . match ( tokenizer . NON_ALPHANUMERIC_REGEXP ) ? true : false ; if ( tokenizer . NUMBERS_MAP [ keyword ] !== undefined && dictionary . get ( keyword ) === undefined ) { dictionary . set ( keyword , values ) ; dictionary . set ( tokenizer . NUMBERS_MAP [ keyword ] , values ) ; return ; } if ( keyword . length === 3 ) { if ( nonAlphanumericChars ) { return ; } if ( dictionaryExtension . get ( keyword ) ) { return ; } dictionaryExtension . set ( keyword , values ) ; } else if ( keyword . length <= 2 ) { if ( keyword . length > 1 && nonAlphanumericChars ) { return ; } if ( dictionary . get ( keyword ) ) { return ; } dictionary . set ( keyword , values ) ; } } ) ; lineReader . on ( 'close' , ( ) => { fs . writeFile ( outputDictionary , JSON . stringify ( [ ... dictionary ] ) , ( err ) => { if ( err ) { console . error ( err ) ; return ; } fs . writeFile ( outputDictionaryExtension , JSON . stringify ( [ ... dictionaryExtension ] ) , ( err2 ) => { err2 ? console . error ( err2 ) : console . log ( 'Done with dictionary and trigrams dictionary extension.' ) ; } ) ; } ) ; } ) ; } ; if ( process && process . argv && process . argv . length > 2 ) { const lang = process . argv [ 2 ] ; if ( lang === 'en' || lang === 'es' ) { createDictionaryFromFastText ( lang ) ; } else { console . error ( 'ERROR: Unknown language:' , lang ) ; process . exit ( 1 ) ; } }