import { expect } from 'chai' ; import * as _ from 'lodash' ; import * as utf8 from 'utf8' ; import { LanguageIdentifier } from '../../src/cldFactory' ; import { loadModule } from '../../src/index' ; describe ( 'cld' , ( ) => { let identifier ; beforeAll ( async ( done ) => { const cldFactory = await loadModule ( ) ; identifier = cldFactory . create ( 0 , 1000 ) ; done ( ) ; } ) ; afterAll ( ( ) => identifier . dispose ( ) ) ; describe ( 'findLanguage' , ( ) => { const fixture = require ( '../__fixtures__/nnet_lang_id_test_data.json' ) ; const excluded = [ 'bs' , 'id' ] ; fixture . filter ( ( { expected } ) => ! _ . includes ( excluded , expected ) ) . forEach ( ( { expected , source } ) => { it ( `should detect ${ expected }` , ( ) => { const result = identifier . findLanguage ( source ) ; expect ( result . language ) . to . equal ( expected ) ; } ) ; } ) ; } ) ; describe ( 'findMostFrequentLanguages' , ( ) => { it ( 'should detect multiple languages with proportions' , ( ) => { const text = 'This piece of text is in English. Този текст е на Български.' ; const expected_bg_span = utf8 . encode ( ' Този текст е на Български ' ) ; const expected_en_span = utf8 . encode ( ' This piece of text is in English ' ) ; const expected_byte_sum = expected_bg_span . length + expected_en_span . length ; const epsilon = 0.00001 ; const num_queried_langs = 3 ; const expected_lang_proportions = [ { expected : 'bg' , span : expected_bg_span . length / expected_byte_sum } , { expected : 'en' , span : expected_en_span . length / expected_byte_sum } ] ; const results = identifier . findMostFrequentLanguages ( text , num_queried_langs ) ; expect ( results ) . to . have . lengthOf ( expected_lang_proportions . length ) ; for ( let idx = 0 ; idx < results . length ; idx ++ ) { const lang = results [ idx ] . language ; expect ( lang ) . to . equal ( expected_lang_proportions [ idx ] . expected ) ; const propotion = Math . abs ( results [ idx ] . proportion - expected_lang_proportions [ idx ] . span ) ; expect ( propotion ) . to . lessThan ( epsilon ) ; const byte_ranges = results [ idx ] . byte_ranges ; expect ( byte_ranges ) . to . have . lengthOf ( 1 ) ; const { start_index , end_index } = byte_ranges [ 0 ] ; const byte_ranges_text = text . substr ( start_index , end_index - start_index ) ; const expected_byte_ranges_span = lang === 'bg' ? 'Този текст е на Български.' : `This piece of text is in English. ` ; expect ( byte_ranges_text ) . to . equal ( expected_byte_ranges_span ) ; } } ) ; } ) ; } ) ;