/**
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License.
 */ import * as BB from 'botbuilder' ; import * as CLM from '@conversationlearner/models' ; import * as DialogEditing from './dialogEditing' ; import * as DialogUtils from './dialogUtils' ; import * as Util from './util' ; import Plain from 'slate-plain-serializer' ; import { REPROMPT_SELF } from '../types/const' ; import { ImportedAction } from '../types/models' ; import { Case } from '../types/obiTypes' ; import { User } from '../types' ; export async function toTranscripts ( appDefinition , appId , user , fetchActivitiesAsync ) { const definitions = { entities : appDefinition . entities , actions : appDefinition . actions , trainDialogs : [ ] } ; return Promise . all ( appDefinition . trainDialogs . map ( td => getActivities ( appId , td , user , definitions , fetchActivitiesAsync ) ) ) ; } export interface OBIImportData { appId : string ; files : File [ ] ; autoCreate : boolean ; autoMerge : boolean ; autoActionCreate : boolean ; } export async function getLogDialogActivities ( appId , logDialogId , user , actions , entities , conversationId , channelId , fetchLogDialogThunkAsync , fetchActivitiesThunkAsync ) { const logDialog = await fetchLogDialogThunkAsync ( appId , logDialogId , true , true , true ) ; if ( ! logDialog ) { throw new Error ( `Unable to find log Id: ${ logDialogId }` ) ; } const trainDialog = CLM . ModelUtils . ToTrainDialog ( logDialog , actions , entities ) ; const teachWithActivities = await fetchActivitiesThunkAsync ( appId , trainDialog , user . name , user . id , false , true ) ; const activities = teachWithActivities . activities ; if ( conversationId || channelId ) { addActivityReferences ( activities , conversationId , channelId ) ; } return activities ; } function addActivityReferences ( activities , conversationId , channelId ) { activities . forEach ( a => { if ( channelId ) { a . channelId = channelId ; } if ( conversationId ) { a . conversation = { id : conversationId } ; } } ) ; } async function getActivities ( appId , trainDialog , user , definitions , fetchActivitiesAsync ) { const newTrainDialog = Util . deepCopy ( trainDialog ) ; newTrainDialog . definitions = definitions ; const teachWithActivities = await fetchActivitiesAsync ( appId , newTrainDialog , user . name , user . id , false ) ; return { activities : teachWithActivities . activities } ; } interface TranscriptActionInput { entityName : string ; } export interface TranscriptActionCall { type : string ; actionName : string ; actionInput : TranscriptActionInput [ ] ; actionOutput : OBIActionOutput [ ] ; } export interface OBIActionOutput { entityName : string ; value ? : string ; } export function isSameActivity ( activity1 , activity2 ) { if ( ( activity1 && ! activity2 ) || ( activity2 && ! activity1 ) || ( activity1 . text !== activity2 . text ) ) { return false ; } const attachments1 = activity1 . attachments ? JSON . stringify ( activity1 . attachments ) : null ; const attachments2 = activity2 . attachments ? JSON . stringify ( activity2 . attachments ) : null ; if ( attachments1 !== attachments2 ) { return false ; } return true ; } export async function lgMapFromLGFiles ( lgFiles , lgItemList ? ) { const map = lgItemList ?? [ ] ; if ( lgFiles ) { for ( const lgFile of lgFiles ) { if ( lgFile . name . endsWith ( '.lg' ) ) { const fileText = await Util . readFileAsync ( lgFile ) ; CLM . ObiUtils . addToLGMap ( fileText , map ) ; } else { throw new Error ( `Expecting .lg file.\n\n Given: ${ lgFile . name }` ) ; } } } return map ; } export function usesLG ( transcript ) { for ( let activity of transcript ) { if ( activity . type === 'message' && activity . from . role === 'bot' ) { let lgName = lgNameFromImportText ( activity . text ) ; if ( lgName ) { return true ; } } } return false ; } export function fromLG ( transcript , lgItemList ) { let usedLG = false ; for ( let activity of transcript ) { if ( activity . type === 'message' && activity . from . role === 'bot' ) { let lgName = lgNameFromImportText ( activity . text ) ; if ( lgName ) { usedLG = true ; let response = lgItemList . find ( lg => lg . lgName === lgName ) ; if ( response ) { activity . text = ( response . suggestions . length > 0 ) ? JSON . stringify ( response ) : response . text ; } else { activity . text = `LG reference "${ lgName }" not found` ; } } } } return usedLG ; } export function toLG ( transcript , lgItemList , entities , actions ) { for ( let activity of transcript ) { if ( activity . channelData ?. clData ?. actionId ) { const lgItem = lgItemList . find ( lg => lg . actionId === activity . channelData . clData . actionId ) ; if ( lgItem ) { activity . text = `[${ lgItem . lgName }]` ; delete activity . attachments ; delete activity . attachmentLayout ; } } } } export async function trainDialogFromTranscriptImport ( transcript , lgItemList , entities , actions , app , createActionThunkAsync ? , createEntityThunkAsync ? ) { const transcriptHash = CLM . hashText ( JSON . stringify ( transcript ) ) ; let trainDialog = { trainDialogId : undefined ! , version : undefined ! , packageCreationId : undefined ! , packageDeletionId : undefined ! , sourceLogDialogId : undefined ! , initialFilledEntities : [ ] , rounds : [ ] , tags : [ ] , description : '' , createdDateTime : new Date ( ) . toJSON ( ) , lastModifiedDateTime : new Date ( ) . toJSON ( ) , validity : CLM . Validity . INVALID , clientData : { importHashes : [ transcriptHash ] } } ; if ( lgItemList ) { fromLG ( transcript , lgItemList ) ; } let curRound = null ; let nextFilledEntities = [ ] ; for ( let index = 0 ; index < transcript . length ; index = index + 1 ) { const activity = transcript [ index ] ; const nextActivity = transcript [ index + 1 ] ; if ( activity . text === "END_SESSION" ) { return trainDialog ; } else if ( ! activity . type || activity . type === "message" ) { if ( activity . from . role === "user" ) { const textVariations = [ { text : activity . text , labelEntities : [ ] } ] ; if ( activity . channelData ?. textVariations ) { activity . channelData . textVariations . forEach ( ( tv ) => { if ( textVariations . length < CLM . MAX_TEXT_VARIATIONS && activity . text !== tv . text ) { let altTextVariation = { text : tv . text , labelEntities : [ ] } ; textVariations . push ( altTextVariation ) ; } } ) ; } let extractorStep = { textVariations : textVariations } ; curRound = { extractorStep , scorerSteps : [ ] } ; trainDialog . rounds . push ( curRound ) ; } else if ( activity . from . role === "bot" ) { let action = findMatchedAction ( activity , entities , actions , nextFilledEntities ) ; let logicResult ; let scoredAction ; let filledEntities = Util . deepCopy ( nextFilledEntities ) ; if ( activity . channelData ?. type === "ActionCall" ) { const actionCall = activity . channelData as TranscriptActionCall ; const isTerminal = ! nextActivity || nextActivity . from . role === "user" ; if ( ! action ) { action = await DialogEditing . getOrCreatePlaceholderAPIAction ( app . appId , actionCall . actionName , isTerminal , actions , createActionThunkAsync as any ) ; } if ( ! action && createActionThunkAsync ) { throw new Error ( "Unable to create API placeholder Action" ) ; } if ( action ) { scoredAction = { actionId : action . actionId , payload : action . payload , isTerminal : action . isTerminal , actionType : CLM . ActionTypes . API_LOCAL , score : 1 } ; const actionFilledEntities = await importActionOutput ( actionCall . actionOutput , entities , app . appId , createEntityThunkAsync ) ; logicResult = { logicValue : undefined , changedFilledEntities : actionFilledEntities , } ; nextFilledEntities = actionFilledEntities ; } } let scoreInput = { filledEntities : filledEntities , context : { } , maskedActions : [ ] } ; let scorerStep = { importText : action ? undefined : activity . text , input : scoreInput , labelAction : action ? action . actionId : CLM . CL_STUB_IMPORT_ACTION_ID , logicResult , scoredAction } ; if ( curRound ) { curRound . scorerSteps . push ( scorerStep ) ; } else { throw new Error ( "Dialogs must start with User turn (role='user')" ) ; } } } } return trainDialog ; } function findMatchedAction ( activity , entities , actions , filledEntities ) { if ( activity . channelData ?. clData ) { const clData = activity . channelData . clData ; if ( clData . actionId ) { const action = actions . find ( a => a . actionId === clData . actionId ) ; if ( action ) { return action ; } else { throw new Error ( "Provided ActionId does not exist in this model" ) ; } } } const lgName = lgNameFromImportText ( activity . text ) ; if ( lgName ) { const action = actions . find ( a => a . clientData ?. lgName === lgName ) ; if ( action ) { return action ; } } const hashText = hashTextFromActivity ( activity , entities , filledEntities ) ; return ( hashText ? findActionFromHashText ( hashText , actions ) : undefined ) ; } export function generateEntityMapForAction ( action , filledEntityMap = new Map < string , string > ( ) ) { const map = new Map < string , string > ( ) ; action . requiredEntities . forEach ( e => { let value = filledEntityMap . get ( e ) ; if ( value ) { map . set ( e , value ) ; } else { map . set ( e , "" ) ; } } ) ; return map ; } export interface ConditionEntityAndValue { entity : string ; value : string ; } export function parseEntityConditionFromDialogCase ( branch , entityConditions ) { if ( ! branch . value ) { throw new Error ( "SwitchCondition cases must have value" ) ; } const tokens = branch . value . split ( "==" ) . map ( ( i ) => { const trimmed = i . trim ( ) ; if ( trimmed . length === 0 ) { throw new Error ( "SwitchCondition entity and value must be non-empty" ) ; } return trimmed ; } ) ; if ( tokens . length !== 2 ) { throw new Error ( "SwitchCondition case is expected to have format 'x == y'" ) ; } const [ entity , value ] = tokens ; let conditionValues = entityConditions [ entity ] ; if ( ! conditionValues ) { conditionValues = new Set < string > ( ) ; entityConditions [ entity ] = conditionValues ; } conditionValues . add ( value ) ; return { entity , value } ; } export function hashTextFromActivity ( activity , entities , filledEntities ) { if ( activity . channelData ?. type === "ActionCall" ) { const actionCall = activity . channelData as TranscriptActionCall ; return actionCall . actionName ; } else if ( filledEntities && filledEntities . length > 0 ) { const filledEntityMap = DialogUtils . filledEntityIdMap ( filledEntities , entities ) ; return importTextWithEntityIds ( activity . text , filledEntityMap ) ; } return activity . text ; } export function importTextWithEntityIds ( importText , valueMap ) { let outText = importText . slice ( ) ; valueMap . forEach ( ( value , entityId ) => { outText = outText . replace ( value , entityId ) ; } ) ; return outText ; } export function replaceImportActions ( trainDialog , actions , entities ) { const actionsWithHash = actions . filter ( a => a . clientData ?. actionHashes && a . clientData . actionHashes . length > 0 ) ; if ( actionsWithHash . length === 0 ) { return false ; } let match = false ; trainDialog . rounds . forEach ( round => { round . scorerSteps . forEach ( scorerStep => { let foundAction = findActionFromScorerStep ( scorerStep , actionsWithHash , entities ) ; if ( foundAction ) { scorerStep . labelAction = foundAction . actionId ; delete scorerStep . importText ; match = true ; } } ) ; } ) ; return match ; } export function expandLGItems ( trainDialog , lgItems ) { for ( const round of trainDialog . rounds ) { for ( const scorerStep of round . scorerSteps ) { const lgName = scorerStep . importText ? lgNameFromImportText ( scorerStep . importText ) : null ; if ( lgName ) { let lgItem = lgItems . find ( lg => lg . lgName === lgName ) ; if ( lgItem ) { scorerStep . importText = lgItem . text ; } } } } } function lgNameFromImportText ( importText ) { return importText . substring ( importText . indexOf ( "[" ) + 1 , importText . lastIndexOf ( "]" ) ) . trim ( ) ; } function findActionFromScorerStep ( scorerStep , actions , entities ) { let hashText = null ; if ( scorerStep . importText ) { const lgName = lgNameFromImportText ( scorerStep . importText ) ; if ( lgName ) { const action = actions . find ( a => a . clientData ?. lgName === lgName ) ; if ( action ) { return action ; } } const filledEntityMap = DialogUtils . filledEntityIdMap ( scorerStep . input . filledEntities , entities ) ; hashText = importTextWithEntityIds ( scorerStep . importText , filledEntityMap ) ; } else if ( scorerStep . labelAction && CLM . ActionBase . isPlaceholderAPI ( scorerStep . scoredAction ) ) { const apiAction = new CLM . ApiAction ( scorerStep . scoredAction as any ) ; hashText = apiAction . name ; } if ( hashText ) { return findActionFromHashText ( hashText , actions ) ; } return undefined ; } /**
 * Holds information about enum entities that should be set in a TrainDialog after a round containing
 * an API action with known output values.
 */ interface EnumDataFromCondition { enumEntityId : string ; enumValueId : string ; enumValueText : string ; } param scorerStepConditions param actionImportIdToExpectedEntityName export async function createImportedActions ( appId , trainDialog , templates , lgItems , actions , entities , scorerStepConditions , actionImportIdToExpectedEntityName , createActionThunkAsync ) { const newActions = [ ] ; for ( const round of trainDialog . rounds ) { for ( const [ scoreIndex , scorerStep ] of round . scorerSteps . entries ( ) ) { if ( ! scorerStep . importText ) { continue ; } let action = findActionFromScorerStep ( scorerStep , [ ... newActions , ... actions ] , [ ] ) ; if ( ! action ) { const isTerminal = round . scorerSteps . length === scoreIndex + 1 ; let importedAction ; if ( lgItems ) { const lgName = lgNameFromImportText ( scorerStep . importText ) ; if ( lgName ) { let lgItem = lgItems . find ( lg => lg . lgName === lgName ) ; if ( lgItem ) { importedAction = { text : lgItem . text , buttons : lgItem . suggestions , isTerminal , reprompt : lgItem . suggestions . length > 0 , lgName } ; } else { lgItem = { lgName : "" , text : "Can't Parse LG" , suggestions : [ ] } ; } } } if ( ! importedAction ) { importedAction = { text : scorerStep . importText , buttons : [ ] , isTerminal , reprompt : false , actionHash : CLM . hashText ( scorerStep . importText ) } ; } action = await createActionFromImport ( appId , importedAction , templates , scorerStep , scorerStepConditions , entities , actionImportIdToExpectedEntityName , createActionThunkAsync ) ; newActions . push ( action ) ; } scorerStep . labelAction = action . actionId ; delete scorerStep . importText ; } } } /**
 * Updates memory state by setting `logicResult` in the `TrainDialog` `TrainScorerSteps` for cases where
 * the dialog produces a deterministic output from an API call.
 */ export function setMemoryStateForImportedTrainDialog ( entities , actions , trainDialog , scorerStepConditions ) { if ( ! scorerStepConditions ) { return ; } for ( const round of trainDialog . rounds ) { setMemoryStateForApiActionWithSwitch ( entities , actions , round . scorerSteps , scorerStepConditions ) ; } } /**
 * Updates memory state for a single `TrainRound`.  Specifically, if we have an API action followed by
 * a `SwitchCondition` node that consumes the output of that API, we know that each branch of the dialog
 * should represent a scenario where the API has returned the given conditional value.  This code sets
 * that conditional value in bot memory.
 */ function setMemoryStateForApiActionWithSwitch ( entities , actions , scorerSteps , scorerStepConditions ) { for ( let i = 0 ; i < scorerSteps . length - 1 ; i = i + 1 ) { if ( ! scorerSteps [ i ] . labelAction ) { continue ; } const j = i + 1 ; if ( j >= scorerSteps . length ) { break ; } const actionId1 = scorerSteps [ i ] . labelAction ; const action1 = actions . find ( a => a . actionId === actionId1 ) ; if ( ! action1 || action1 . actionType !== CLM . ActionTypes . API_LOCAL ) { continue ; } const scorerStep = scorerSteps [ i ] ; const nextScorerStep = scorerSteps [ j ] ; if ( ! nextScorerStep . importId || ! scorerStepConditions [ nextScorerStep . importId ] ) { continue ; } if ( ! scorerStep . logicResult ) { scorerStep . logicResult = { logicValue : undefined , changedFilledEntities : [ ] } ; } const conditions = scorerStepConditions [ nextScorerStep . importId ] ; const enumConditionData = conditions . map ( condition => getEnumConditionData ( entities , condition ) ) ; for ( const conditionEntity of enumConditionData ) { const filledEntity = { entityId : conditionEntity . enumEntityId , values : [ { userText : conditionEntity . enumValueText , displayText : conditionEntity . enumValueText , builtinType : null , resolution : null , enumValueId : conditionEntity . enumValueId } ] } ; scorerStep . logicResult . changedFilledEntities . push ( filledEntity ) ; } } } throws function getEnumConditionData ( entities , condition ) { if ( ! condition . valueId ) { throw new Error ( `Action condition doesn't reference an entity` ) ; } const conditionEntity = entities . find ( e => e . entityId === condition . entityId ) ; if ( ! conditionEntity ) { throw new Error ( `Couldn't find entity with id ${ condition . entityId }` ) ; } if ( conditionEntity . entityType !== CLM . EntityType . ENUM || ! conditionEntity . enumValues ) { throw new Error ( `Entity ${ conditionEntity . entityId } is not a valid enum` ) ; } const enumValue = conditionEntity . enumValues . find ( val => val . enumValueId === condition . valueId ) ; if ( ! enumValue ) { throw new Error ( `Enum entity ${ conditionEntity . entityName } missing enum value ${ condition . valueId }` ) ; } return { enumEntityId : condition . entityId , enumValueId : condition . valueId , enumValueText : enumValue . enumValue , } ; } param scorerStepConditions param actionImportIdToExpectedEntityName async function createActionFromImport ( appId , importedAction , templates , scorerStep , scorerStepConditions , entities , actionImportIdToExpectedEntityName , createActionThunkAsync ) { const template = DialogUtils . bestTemplateMatch ( importedAction , templates ) ; const actionType = template ? CLM . ActionTypes . CARD : CLM . ActionTypes . TEXT ; const repromptActionId = template && importedAction . buttons . length > 0 ? REPROMPT_SELF : undefined ; const textValue = Plain . deserialize ( `${ importedAction . text }` ) ; let payload = null ; if ( template ) { const actionArguments = [ ] ; const textBody = template . variables . find ( v => v . type === "TextBody" || v . type === "TextBlock" ) ; if ( textBody ) { const title = Plain . deserialize ( importedAction . text ) ; actionArguments . push ( { parameter : textBody . key , value : { json : title . toJSON ( ) } } ) ; } if ( importedAction . buttons . length > 0 ) { const buttons = importedAction . buttons . map ( t => Plain . deserialize ( t ) ) ; buttons . forEach ( ( button , index ) => { if ( ( index + 1 ) < template . variables . length ) { actionArguments . push ( { parameter : template . variables [ index + 1 ] . key , value : { json : button . toJSON ( ) } } ) ; } } ) ; } const cp = { payload : template . name , arguments : actionArguments } ; payload = JSON . stringify ( cp ) ; } else { const tp = { json : textValue . toJSON ( ) } ; payload = JSON . stringify ( tp ) ; } const action = new CLM . ActionBase ( { actionId : null ! , payload , createdDateTime : new Date ( ) . toJSON ( ) , isTerminal : importedAction . isTerminal , repromptActionId , requiredEntitiesFromPayload : [ ] , requiredEntities : [ ] , negativeEntities : [ ] , requiredConditions : [ ] , negativeConditions : [ ] , suggestedEntity : undefined , version : 0 , packageCreationId : 0 , packageDeletionId : 0 , actionType , entityId : undefined , enumValueId : undefined , clientData : { actionHashes : importedAction . actionHash ? [ importedAction . actionHash ] : [ ] , lgName : importedAction . lgName } } ) ; if ( scorerStep . importId ) { const importId = scorerStep . importId ; if ( scorerStepConditions ?. [ importId ] ) { action . requiredConditions = scorerStepConditions [ importId ] ; } if ( actionImportIdToExpectedEntityName ?. [ importId ] ) { let entity = entities . find ( e => e . entityName === actionImportIdToExpectedEntityName ?. [ importId ] ) ; if ( entity ) { action . suggestedEntity = entity . entityId ; } } } const newAction = await createActionThunkAsync ( appId , action ) ; if ( ! newAction ) { throw new Error ( "Unable to create action" ) ; } return newAction ; } export function importedActionFromImportText ( importText , isTerminal ) { try { const lgItem = JSON . parse ( importText ) ; return { text : lgItem . text , buttons : lgItem . suggestions , isTerminal , reprompt : lgItem . suggestions . length > 0 } ; } catch ( e ) { return { text : importText , buttons : [ ] , isTerminal , reprompt : false } ; } } export function findActionFromHashText ( hashText , actions ) { const importHash = CLM . hashText ( hashText ) ; let matchedActions = actions . filter ( a => { return a . clientData ?. actionHashes ?. includes ( importHash ) ; } ) ; if ( matchedActions . length > 1 ) { matchedActions = matchedActions . filter ( ma => ! CLM . ActionBase . isPlaceholderAPI ( ma ) ) ; } return matchedActions [ 0 ] ; } export function areTranscriptsEqual ( transcript1 , transcript2 , excessOk = false ) { if ( ! excessOk && transcript1 . length !== transcript2 . length ) { return false ; } if ( transcript1 . length === 0 || transcript2 . length === 0 ) { throw new Error ( "Transcript has no turns" ) ; } if ( ! transcript1 [ 0 ] . conversation || ! transcript2 [ 0 ] . conversation ) { throw new Error ( "Not a valid transcript. Conversation not defined" ) ; } if ( transcript1 [ 0 ] . conversation . id !== transcript2 [ 0 ] . conversation . id ) { throw new Error ( "Not a valid comparison.  ConversationIds do not match." ) ; } if ( transcript1 [ 0 ] . channelId === transcript2 [ 0 ] . channelId ) { throw new Error ( "Not a valid comparison.  Same channel." ) ; } for ( let i = 0 ; i < Math . min ( transcript1 . length , transcript2 . length ) ; i = i + 1 ) { const activity1 = transcript1 [ i ] ; const activity2 = transcript2 [ i ] ; if ( activity1 . type !== activity2 . type ) { return false ; } if ( ! activity1 . from || ! activity2 . from ) { throw new Error ( "Not a valid transcript.  Has no from" ) ; } if ( activity1 . from . role !== activity2 . from . role ) { return false ; } if ( activity1 . text !== activity2 . text ) { if ( activity1 . from . role === "user" ) { throw new Error ( "Not a valid comparison.  Inconsistent User Input" ) ; } else { return false ; } } } return true ; } /**
 * Given the {entityName, entityValue} results from some action being imported, returns
 * an array of FilledEntity instances representing those results.
 * Entities that do not yet exist will be created and added to `entities`.
 */ export async function importActionOutput ( actionResults , entities , appId , createEntityThunkAsync ? ) { const filledEntities = [ ] ; for ( const actionResult of actionResults ) { const foundEntity = entities . find ( e => e . entityName === actionResult . entityName ) ; let entityId = "" ; if ( foundEntity ) { entityId = foundEntity . entityId ; } else if ( createEntityThunkAsync ) { const newEntity = { entityId : undefined ! , entityName : actionResult . entityName , resolverType : "none" , createdDateTime : new Date ( ) . toJSON ( ) , lastModifiedDateTime : new Date ( ) . toJSON ( ) , isResolutionRequired : false , isMultivalue : false , isNegatible : false , negativeId : null , positiveId : null , entityType : CLM . EntityType . LOCAL , version : null , packageCreationId : null , packageDeletionId : null , doNotMemorize : false } ; entityId = await ( ( createEntityThunkAsync ( appId , newEntity ) as any ) as Promise < string > ) ; if ( ! entityId ) { throw new Error ( "Invalid Entity Definition" ) ; } newEntity . entityId = entityId ; entities . push ( newEntity ) ; } else { entityId = "UNKNOWN ENTITY" ; } const memoryValue = { userText : actionResult . value ? actionResult . value : null , displayText : null , builtinType : null , resolution : { } } ; const filledEntity = { entityId , values : [ memoryValue ] } ; filledEntities . push ( filledEntity ) ; } return filledEntities ; }