/**
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License.
 */ import * as CLM from '@conversationlearner/models' ; import { TreeUtils } from './TreeUtils' ; export interface LogScore { logDialogId : string ; knownPath ? : boolean ; validEndAction ? : boolean ; numberUnknownUtterances ? : number ; numberLowConfidence ? : number ; score : number ; } const LOW_CONFIDENCE_THRESHOLD = 0.2 ; function calcScore ( knownPath , validEndAction , numberUnknownUtterances , numberLowConfidence ) { let score = ( knownPath ? 0 : 1 ) + ( validEndAction ? 0 : 1 ) + Math . min ( 1 , numberUnknownUtterances * 0.1 ) + Math . min ( 1 , numberLowConfidence * 0.1 ) ; return + ( score / 4 ) . toFixed ( 4 ) ; } function isLastActionInSet ( trainDialog , actionIds ) { const lastRound = trainDialog . rounds [ trainDialog . rounds . length - 1 ] ; if ( lastRound . scorerSteps . length === 0 ) { return false ; } const actionId = lastRound . scorerSteps [ lastRound . scorerSteps . length - 1 ] . labelAction ; if ( ! actionId ) { return false ; } return actionIds . includes ( actionId ) ; } function calcNumberUnknownUtterances ( trainDialog , knownUtterances ) { const utterances = getUtterances ( trainDialog ) ; return utterances . reduce ( ( acc , u ) => acc + ( knownUtterances . includes ( u ) ? 0 : 1 ) , 0 ) ; } function calcNumberLowConfidence ( logDialog , knownUtterances ) { let numLowConfidence = 0 ; logDialog . rounds . forEach ( r => r . scorerSteps . forEach ( ss => { if ( ss . predictionDetails . scoredActions . length >= 2 ) { const scoredActions = ss . predictionDetails . scoredActions . sort ( sa => sa . score ) ; const scoreDiff = scoredActions [ 0 ] . score - scoredActions [ 1 ] . score ; if ( scoreDiff < LOW_CONFIDENCE_THRESHOLD ) { numLowConfidence = numLowConfidence + 1 ; } } } ) ) ; return numLowConfidence ; } function getAllUtterances ( trainDialogs ) { const utterances = [ ] ; trainDialogs . forEach ( td => td . rounds . forEach ( r => r . extractorStep . textVariations . forEach ( t => utterances . push ( removePunctuationAndLowercase ( t . text ) ) ) ) ) ; return [ ... new Set ( utterances ) ] ; } function getUtterances ( trainDialog ) { const utterances = [ ] ; trainDialog . rounds . forEach ( r => r . extractorStep . textVariations . forEach ( t => utterances . push ( removePunctuationAndLowercase ( t . text ) ) ) ) ; return utterances ; } function removePunctuationAndLowercase ( text ) { const regex = /[!"#$%&'ʼ()*+,-./:;<=>?@[\]^_`{|}~]/g ; return text . replace ( regex , '' ) . toLowerCase ( ) ; } param logDialogs param trainDialogs param entities export function rankLogs ( logDialogs , trainDialogs , entities ) { const trainTree = new TreeUtils ( trainDialogs , entities ) ; const leafActionIds = trainTree . getLeafActionIds ( ) ; const allKnownUtterances = getAllUtterances ( trainDialogs ) ; const logScores = [ ] ; logDialogs . forEach ( ld => { const convertedDialog = CLM . ModelUtils . ToTrainDialog ( ld ) ; const knownPath = trainTree . isInTree ( convertedDialog ) ; const validEndAction = isLastActionInSet ( convertedDialog , leafActionIds ) ; const numberUnknownUtterances = calcNumberUnknownUtterances ( convertedDialog , allKnownUtterances ) ; const numberLowConfidence = calcNumberLowConfidence ( ld , allKnownUtterances ) ; const score = calcScore ( knownPath , validEndAction , numberUnknownUtterances , numberLowConfidence ) ; logScores . push ( { logDialogId : ld . logDialogId , knownPath , validEndAction , numberUnknownUtterances , numberLowConfidence , score } ) ; } ) ; return logScores ; }