import * as tf from '@tensorflow/tfjs' ; import '@tensorflow/tfjs-node' ; import { merge } from 'lodash' ; import { AidaPipeline , defaultPipelineDefinition } from './pipelines/zebraWings/pipeline' ; import * as types from './types' ; const logger = { debug : ( ) => null , error : console . error , log : console . log , warn : console . warn } ; const trainTestAndSaveModels = async ( ) => { const pretrainedNGramVectors = new Map < string , Float32Array > ( require ( '../public/models/dictionary.json' ) ) ; const ngramToIdDictionary = require ( '../public/models/ngram_to_id_dictionary.json' ) ; const datasetParams = require ( '../public/models/dataset_params.json' ) ; const trainDataset = require ( '../public/models/dataset_training.json' ) ; const testDataset = require ( '../public/models/dataset_testing.json' ) ; const pipelineDefinition = merge ( { } , defaultPipelineDefinition , { config : { default : { batchSize : 120 } } } ) ; const pipeline = new AidaPipeline ( { datasetParams , logger , ngramToIdDictionary , pipelineDefinition , pretrainedNGramVectors } ) ; await pipeline . train ( trainDataset ) ; const stats = await pipeline . test ( testDataset ) ; logger . log ( stats ) ; await pipeline . save ( { classificationPath : 'file://public/models/pretrained/node/classification' , embeddingPath : 'file://public/models/pretrained/node/embedding' , nerPath : 'file://public/models/pretrained/node/ner' } ) ; return pipeline ; } ; const loadSavedModels = async ( ) => { const ngramToIdDictionary = require ( '../public/models/ngram_to_id_dictionary.json' ) ; const datasetParams = require ( '../public/models/dataset_params.json' ) ; const pretrainedEmbedding = await tf . loadLayersModel ( 'file://public/models/pretrained/node/embedding/model.json' ) ; const pretrainedClassifier = await tf . loadLayersModel ( 'file://public/models/pretrained/node/classification/model.json' ) ; const pretrainedNer = await tf . loadLayersModel ( 'file://public/models/pretrained/node/ner/model.json' ) ; const pipeline = new AidaPipeline ( { datasetParams , logger , ngramToIdDictionary , pretrainedClassifier , pretrainedEmbedding , pretrainedNer } ) ; const testDataset = require ( '../public/models/dataset_testing.json' ) ; const stats = await pipeline . test ( testDataset ) ; logger . log ( stats ) ; return pipeline ; } ; trainTestAndSaveModels ( ) ;