import { tensor } from '@tensorflow/tfjs-node' ; import { Sample } from '../../../dataset/input-data-parser' ; import { LabelEncoder } from '../../../encode/label-encoder' ; import { OneHotEncoder } from '../../../encode/one-hot-encoder' ; import { PADDING_TOKEN , UNKNOWN_TOKEN } from '../../../preprocess/constants' ; import { Preprocessor } from '../../../preprocess/preprocessor' ; import { InputData , OutputData } from './types' ; export class Processor { constructor ( readonly preprocessor , readonly tokensEncoder , readonly intentEncoder ) { } processSamples ( samples ) { const texts = samples . map ( sample => sample . text ) ; const intents = samples . map ( sample => sample . intent ) ; return { x : this . processTexts ( texts ) , y : this . generateOutput ( intents ) } ; } processTexts ( texts ) { return tensor ( texts . map ( text => this . processText ( text ) ) ) ; } private processText ( text ) { const sequence = this . generateSequence ( text ) ; return this . tokensEncoder . encode ( sequence ) ; } private generateSequence ( text ) { const sequence = this . preprocessor . preprocess ( text , PADDING_TOKEN ) ; return this . maskUnknownTokens ( sequence ) ; } private maskUnknownTokens ( sequence ) { return sequence . map ( token => this . tokensEncoder . items . includes ( token ) ? token : UNKNOWN_TOKEN ) ; } private generateOutput ( intents ) { return tensor ( this . intentEncoder . encode ( intents ) ) ; } }