/**
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License.
 */ import * as Util from '../Utils/util' ; import * as BB from 'botbuilder' ; import * as OBIUtils from '../Utils/obiUtils' ; import * as CLM from '@conversationlearner/models' ; export enum ComparisonResultType { ALL = 'ALL' , REPRODUCED = 'REPRODUCED' , CHANGED = 'CHANGED' , INVALID_TRANSCRIPT = 'INVALID_TRANSCRIPT' , NO_TRANSCRIPT = 'NOTRANSCRIPT' } export interface TestItem { sourceName : string ; conversationId : string ; transcript ? : BB . Activity [ ] ; ranking ? : number ; logDialogId : string | null ; invalidTranscript ? : boolean ; } export interface SourceComparison { conversationId : string ; sourceNames : [ string , string ] ; result : ComparisonResultType ; } export enum RatingResult { FIRST = 'FIRST' , SECOND = 'SECOND' , SAME = 'SAME' , UNKNOWN = 'UNKNOWN' , NO_TRANSCRIPT = 'NO_TRANSCRIPT' } export interface RatingPair { conversationId : string ; sourceNames : [ string , string ] ; result : RatingResult ; } export class TestSet { appId ? : string ; fileName ? : string ; items : TestItem [ ] ; sourceNames : string [ ] ; ratingPairs : RatingPair [ ] ; lgItems : CLM . LGItem [ ] ; usesLgMap : Map < string , boolean > ; private comparisons : SourceComparison [ ] ; static Create ( source ? ) { let init = Util . deepCopy ( source ) ; return new TestSet ( init ) ; } static Deserialize ( fileText ) { const set = JSON . parse ( fileText , Util . mapReviver ) ; const testSet = this . Create ( set ) ; if ( testSet . items . length === 0 ) { throw new Error ( "No test results found in file" ) ; } return testSet ; } serialize ( ) { return new Blob ( [ JSON . stringify ( this , Util . mapReplacer ) ] , { type : "text/plain;charset=utf-8" } ) ; } numConversations ( ) { const conversationIds = this . items . map ( i => i . conversationId ) ; return [ ... new Set ( conversationIds ) ] . length ; } usesLG ( ) { let temp = Array . from ( this . usesLgMap . values ( ) ) . find ( v => v ) ; return temp || false ; } getTranscripts ( sourceName ) { return this . items . filter ( i => i . sourceName === sourceName ) . map ( i => i . transcript ) . filter ( i => i !== undefined ) as BB . Activity [ ] [ ] ; } getTranscript ( sourceName , conversationId ) { const item = this . items . find ( i => i . sourceName === sourceName && i . conversationId === conversationId ) ; return item ? item . transcript : undefined ; } getTestItem ( sourceName , conversationId ) { return this . items . find ( i => i . sourceName === sourceName && conversationId === i . conversationId ) ; } getTestItems ( sourceName , conversationIds ? ) { return this . items . filter ( i => i . sourceName === sourceName && ( ! conversationIds || conversationIds . includes ( i . conversationId ) ) ) ; } getAllConversationIds ( ) { const conversationIds = this . items . map ( i => i . conversationId ) ; return [ ... new Set ( conversationIds ) ] ; } getComparisonConversationIds ( pivotName , sourceName , resultType ) { return this . comparisons . filter ( c => c . result === resultType && c . sourceNames . includes ( pivotName ) && c . sourceNames . includes ( sourceName ) ) . map ( c => c . conversationId ) ; } async addLGFiles ( lgFiles ) { this . lgItems = await OBIUtils . lgMapFromLGFiles ( lgFiles , this . lgItems ) ; } async addTranscriptFiles ( transcriptFiles ) { for ( const file of transcriptFiles ) { let fileContent = await Util . readFileAsync ( file ) ; const transcript = JSON . parse ( fileContent ) ; let transcriptUsesLG = OBIUtils . usesLG ( transcript ) ; const sourceName = this . sourceName ( transcript ) ; const conversationId = this . conversationId ( transcript ) ; const item = { sourceName : sourceName , conversationId , logDialogId : null , ranking : undefined , transcript } ; this . addTestItem ( item ) ; if ( transcriptUsesLG ) { this . usesLgMap . set ( sourceName , transcriptUsesLG ) ; } if ( ! this . sourceNames . includes ( sourceName ) ) { this . sourceNames . push ( sourceName ) ; this . sourceNames . sort ( ) ; } } } addTestItem ( item ) { if ( ! item . transcript || item . transcript . length === 0 ) { throw new Error ( `Transcript has no rounds. ConversationID = ${ item . conversationId }` ) ; } if ( ! item . transcript [ 0 ] . conversation || ! item . transcript [ 0 ] . conversation . id ) { throw new Error ( "Transcript does not have a conversationId" ) ; } this . items = this . items . filter ( i => ! ( i . conversationId === item . conversationId && i . sourceName === item . sourceName ) ) ; this . items . push ( item ) ; if ( ! this . sourceNames . includes ( item . sourceName ) ) { this . sourceNames . push ( item . sourceName ) ; this . sourceNames . sort ( ) ; } } compareAll ( ) { this . comparisons = [ ] ; for ( const outerSource of this . sourceNames ) { for ( const innerSource of this . sourceNames ) { if ( innerSource !== outerSource ) { const comparisons = this . getSourceComparisons ( innerSource , outerSource ) ; const allConversationIds = this . getAllConversationIds ( ) ; for ( const conversationId of allConversationIds ) { if ( ! comparisons . find ( c => c . conversationId === conversationId ) ) { let innerItem = this . items . find ( i => i . sourceName === innerSource && i . conversationId === conversationId ) ; let outerItem = this . items . find ( i => i . sourceName === outerSource && i . conversationId === conversationId ) ; if ( ! innerItem || ! innerItem . transcript || ! outerItem || ! outerItem . transcript ) { this . comparisons . push ( { conversationId , sourceNames : [ innerSource , outerSource ] , result : ComparisonResultType . NO_TRANSCRIPT } ) ; } else { let result ; try { result = OBIUtils . areTranscriptsEqual ( innerItem . transcript , outerItem . transcript , true ) ? ComparisonResultType . REPRODUCED : ComparisonResultType . CHANGED ; } catch { result = ComparisonResultType . INVALID_TRANSCRIPT ; } this . comparisons . push ( { conversationId , sourceNames : [ innerSource , outerSource ] , result } ) ; } } } } } } } initRating ( ) { const possiblePairs = [ ] ; for ( const outerSource of this . sourceNames ) { for ( const innerSource of this . sourceNames ) { if ( innerSource !== outerSource ) { const sourceNames = [ outerSource , innerSource ] . sort ( ) ; if ( ! possiblePairs . find ( p => p . sourceNames [ 0 ] === sourceNames [ 0 ] && p . sourceNames [ 1 ] === sourceNames [ 1 ] ) ) { possiblePairs . push ( { conversationId : '' , sourceNames : [ sourceNames [ 0 ] , sourceNames [ 1 ] ] , result : RatingResult . UNKNOWN } ) ; } } } } this . ratingPairs = [ ] ; const conversationIds = this . getAllConversationIds ( ) ; for ( const conversationId of conversationIds ) { const newPairs = Util . deepCopy ( possiblePairs ) ; for ( const ratingPair of newPairs ) { ratingPair . conversationId = conversationId ; } this . ratingPairs = this . ratingPairs . concat ( newPairs ) ; } for ( const ratingPair of this . ratingPairs ) { const comparison = this . getSourceComparisons ( ratingPair . sourceNames [ 0 ] , ratingPair . sourceNames [ 1 ] ) . find ( c => c . conversationId === ratingPair . conversationId ) ; if ( comparison ) { if ( comparison . result === ComparisonResultType . REPRODUCED ) { ratingPair . result = RatingResult . SAME ; } else if ( comparison . result === ComparisonResultType . NO_TRANSCRIPT || comparison . result === ComparisonResultType . INVALID_TRANSCRIPT ) { ratingPair . result = RatingResult . NO_TRANSCRIPT ; } } } this . calcRankings ( ) ; return this . ratingPairs . filter ( rp => rp . result === RatingResult . UNKNOWN ) . length ; } numRatingsNeeded ( ) { return this . ratingPairs . filter ( rp => rp . result === RatingResult . UNKNOWN ) . length ; } getRating ( pivotSource , source , conversationId ) { const ratingPair = this . getRatingPair ( pivotSource , source , conversationId ) ; return ratingPair ? ratingPair . result : RatingResult . UNKNOWN ; } getRatingPair ( pivotSource , source , conversationId ) { const sourceNames = [ pivotSource , source ] . sort ( ) ; return this . ratingPairs . find ( rp => rp . conversationId === conversationId && rp . sourceNames [ 0 ] === sourceNames [ 0 ] && rp . sourceNames [ 1 ] === sourceNames [ 1 ] ) ; } addRatingResult ( pairAB ) { const sourceA = pairAB . sourceNames [ 0 ] ; const sourceB = pairAB . sourceNames [ 1 ] ; const conversationId = pairAB . conversationId ; this . ratingPairs = this . ratingPairs . filter ( rp => ! ( rp . conversationId === pairAB . conversationId && rp . sourceNames [ 0 ] === sourceA && rp . sourceNames [ 1 ] === sourceB ) ) ; this . ratingPairs . push ( pairAB ) ; const checkPairs = this . ratingPairs . filter ( rp => rp . conversationId === pairAB . conversationId ) ; checkPairs . forEach ( rp => { let sourceC ; if ( rp . sourceNames . includes ( sourceA ) && ! rp . sourceNames [ 1 ] . includes ( sourceB ) ) { sourceC = rp . sourceNames [ 1 ] ; } else if ( ! rp . sourceNames . includes ( sourceA ) && rp . sourceNames [ 1 ] . includes ( sourceB ) ) { sourceC = rp . sourceNames [ 0 ] ; } if ( sourceC ) { const pairAC = this . getRatingPair ( sourceA , sourceC , conversationId ) ; const pairBC = this . getRatingPair ( sourceB , sourceC , conversationId ) ; if ( pairAC && pairBC ) { if ( pairAB . result === RatingResult . SAME && pairAC . result === RatingResult . SAME ) { pairBC . result = RatingResult . SAME ; } if ( pairAB . result === RatingResult . SAME && pairBC . result === RatingResult . SAME ) { pairAC . result = RatingResult . SAME ; } const betterAB = this . whichIsBetter ( sourceA , sourceB , conversationId ) ; const betterAC = this . whichIsBetter ( sourceA , sourceC , conversationId ) ; const betterBC = this . whichIsBetter ( sourceB , sourceC , conversationId ) ; if ( betterAB === sourceA && pairBC . result === RatingResult . SAME ) { this . setFirstIsBetter ( sourceA , sourceC , conversationId ) ; } if ( betterAB === sourceB && pairBC . result === RatingResult . SAME ) { this . setFirstIsBetter ( sourceC , sourceA , conversationId ) ; } if ( betterAB === sourceA && betterAC === sourceC ) { this . setFirstIsBetter ( sourceC , sourceB , conversationId ) ; } if ( betterAB === sourceA && betterBC === sourceB ) { this . setFirstIsBetter ( sourceA , sourceC , conversationId ) ; } if ( betterAB === sourceB && betterAC === sourceA ) { this . setFirstIsBetter ( sourceB , sourceC , conversationId ) ; } if ( betterAB === sourceB && betterBC === sourceC ) { this . setFirstIsBetter ( sourceC , sourceA , conversationId ) ; } } } } ) ; } calcRankings ( ) { for ( const item of this . items ) { const ratingPairs = this . ratingPairs . filter ( c => c . conversationId === item . conversationId ) ; item . ranking = 0 ; for ( const ratingPair of ratingPairs ) { if ( ratingPair . result === RatingResult . FIRST && ratingPair . sourceNames [ 0 ] === item . sourceName ) { item . ranking = item . ranking ? item . ranking + 1 : 1 ; } else if ( ratingPair . result === RatingResult . SECOND && ratingPair . sourceNames [ 1 ] === item . sourceName ) { item . ranking = item . ranking ? item . ranking + 1 : 1 ; } } } } unratableConversationIds ( source , pivotSource ? ) { const filterConversationIds = pivotSource ? this . items . filter ( i => i . sourceName === pivotSource ) . map ( i => i . conversationId ) : this . getAllConversationIds ( ) ; const conversationIds = this . items . filter ( i => i . sourceName === source ) . map ( i => i . conversationId ) ; return filterConversationIds . filter ( id => ! conversationIds . includes ( id ) ) ; } unratedConversationIds ( source , filterSource ? ) { const filterConversationIds = filterSource ? this . items . filter ( i => i . sourceName === filterSource ) . map ( i => i . conversationId ) : this . getAllConversationIds ( ) ; const sourceConversationIds = this . items . filter ( i => i . sourceName === source ) . map ( i => i . conversationId ) ; const sharedConversationIds = filterConversationIds . filter ( id => sourceConversationIds . includes ( id ) ) ; const sourceNames = [ filterSource , source ] . sort ( ) ; return this . ratingPairs . filter ( rp => rp . result === RatingResult . UNKNOWN && ( ( filterSource === undefined && rp . sourceNames . includes ( source ) ) || ( rp . sourceNames [ 0 ] === sourceNames [ 0 ] && rp . sourceNames [ 1 ] === sourceNames [ 1 ] ) ) && sharedConversationIds . includes ( rp . conversationId ) ) . map ( rp => rp . conversationId ) ; } getNeededRating ( ) { const neededPairs = this . ratingPairs . filter ( rp => rp . result === RatingResult . UNKNOWN ) ; if ( neededPairs . length === 0 ) { return undefined ; } const index = Util . randomInt ( 0 , neededPairs . length - 1 ) ; return Util . deepCopy ( neededPairs [ index ] ) ; } getSourceComparisons ( sourceName1 , sourceName2 ) { return this . comparisons . filter ( c => c . sourceNames . includes ( sourceName1 ) && c . sourceNames . includes ( sourceName2 ) ) ; } conversationId ( transcript ) { if ( transcript . length === 0 || ! transcript [ 0 ] . conversation || ! transcript [ 0 ] . conversation . id ) { throw new Error ( "Transcript does not have a conversationId" ) ; } return transcript [ 0 ] . conversation . id ; } sourceName ( transcript ) { if ( transcript . length === 0 || ! transcript [ 0 ] . channelId ) { throw new Error ( `Transcript ${ this . conversationId ( transcript ) } does not have a channelId` ) ; } return transcript [ 0 ] . channelId ; } whichIsBetter ( better , worse , conversationId ) { const ratingPair = this . getRatingPair ( better , worse , conversationId ) ; if ( ratingPair ) { if ( ratingPair . result === RatingResult . FIRST ) { return ratingPair . sourceNames [ 0 ] ; } else if ( ratingPair . result === RatingResult . SECOND ) { return ratingPair . sourceNames [ 1 ] ; } } return null ; } setFirstIsBetter ( better , worse , conversationId ) { const ratingPair = this . getRatingPair ( better , worse , conversationId ) ; if ( ratingPair ) { if ( better . localeCompare ( worse ) <= 0 ) { ratingPair . result = RatingResult . FIRST ; } else { ratingPair . result = RatingResult . SECOND ; } } } private constructor ( init ? ) { Object . assign ( this , init ) ; if ( ! this . sourceNames ) { this . sourceNames = [ ] ; } if ( ! this . items ) { this . items = [ ] ; } if ( ! this . comparisons ) { this . comparisons = [ ] ; } if ( ! this . ratingPairs ) { this . ratingPairs = [ ] ; } if ( ! this . lgItems ) { this . lgItems = [ ] ; } if ( ! this . usesLgMap ) { this . usesLgMap = new Map < string , boolean > ( ) ; } } }