import { IDefaultDataset } from 'chatito/dist/adapters/web' ; import { ISentenceTokens } from 'chatito/dist/types' ; import { flatten , shuffle } from 'lodash' ; import { IAidaTokenizer , IDictionariesFromDataset , IDictJsonItem , IPretrainedDictionary } from '../types' ; export function dictionariesFromDataset ( training , testing , tokenizer , language ) { const ret = { intents : Object . keys ( training ) , intentsWithSlots : [ ] , language , maxWordsPerSentence : 0 , slotsToId : { O : 0 } , testX : [ ] , testY : [ ] , testY2 : [ ] , trainX : [ ] , trainY : [ ] , trainY2 : [ ] } ; const intentTrainingStats = new Array ( ret . intents . length ) . fill ( 0 ) ; const intentTestingStats = new Array ( ret . intents . length ) . fill ( 0 ) ; let processedTrainingSentences = [ ] ; let processedTestingSentences = [ ] ; const intentsWithSlotsSet = new Set < string > ( ) ; ret . intents . forEach ( intent => { let containsSlots = false ; const getProcessedSentence = ( sentenceTokens ) => { const y2Tags = sentenceTokens . map ( token => { const words = tokenizer . splitSentenceToWords ( token . value ) ; const encodedSentenceFrag = new Array ( words . length ) . fill ( 0 ) ; const slotName = token . slot ; if ( slotName ) { intentsWithSlotsSet . add ( intent ) ; if ( ! containsSlots ) { containsSlots = true ; } const internalKey = slotName ; if ( ret . slotsToId [ internalKey ] === undefined ) { ret . slotsToId [ internalKey ] = Object . keys ( ret . slotsToId ) . length ; } words . forEach ( ( w , i ) => { encodedSentenceFrag [ i ] = ret . slotsToId [ internalKey ] ; } ) ; } return encodedSentenceFrag ; } ) ; const tagsForSentence = flatten ( y2Tags ) ; const sentence = sentenceTokens . map ( d => d . value ) . join ( '' ) ; const sentenceWords = tokenizer . splitSentenceToWords ( sentence ) ; if ( ret . maxWordsPerSentence < sentenceWords . length ) { ret . maxWordsPerSentence = sentenceWords . length ; } const intentId = ret . intents . findIndex ( k => k === intent ) ; return { sentence , intentId , tagsForSentence } ; } ; const intentProcessedSentences = training [ intent ] . map ( getProcessedSentence ) ; processedTrainingSentences = processedTrainingSentences . concat ( intentProcessedSentences ) ; const intentTestingProcessedSentences = testing [ intent ] . map ( getProcessedSentence ) ; processedTestingSentences = processedTestingSentences . concat ( intentTestingProcessedSentences ) ; } ) ; ret . intentsWithSlots = Array . from ( intentsWithSlotsSet ) ; shuffle ( processedTrainingSentences ) . forEach ( s => { intentTrainingStats [ s . intentId ] ++ ; ret . trainX . push ( s . sentence ) ; ret . trainY . push ( s . intentId ) ; ret . trainY2 . push ( s . tagsForSentence ) ; } ) ; shuffle ( processedTestingSentences ) . forEach ( s => { intentTestingStats [ s . intentId ] ++ ; ret . testX . push ( s . sentence ) ; ret . testY . push ( s . intentId ) ; ret . testY2 . push ( s . tagsForSentence ) ; } ) ; const stats = ret . intents . map ( ( intentKey , index ) => ( { intent : intentKey , testing : intentTestingStats [ index ] , training : intentTrainingStats [ index ] } ) ) ; return { dictionary : ret , stats } ; } export function buildDictionary ( dictJson ) { const dictionaryCache = { NGRAM_TO_ID_MAP : { } , PRETRAINED : new Map ( ) } as IPretrainedDictionary ; dictionaryCache . PRETRAINED = new Map ( dictJson ) ; [ ... dictionaryCache . PRETRAINED . keys ( ) ] . forEach ( ( word , id ) => { dictionaryCache . NGRAM_TO_ID_MAP [ word ] = id ; } ) ; return dictionaryCache ; }