/**
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License.
 */ import * as CLM from '@conversationlearner/models' ; import * as DialogEditing from './dialogEditing' ; import * as OBITypes from '../types/obiTypes' ; import * as OBIUtils from './obiUtils' ; import * as Util from './util' ; import * as fspath from 'path' ; import * as stripJsonComments from 'strip-json-comments' ; import { NONE_RESOLVER_KEY } from '../types/const' ; enum OBIStepType { BEGIN_DIALOG = "Microsoft.BeginDialog" , END_DIALOG = "Microsoft.EndDialog" , END_TURN = "Microsoft.EndTurn" , HTTP_REQUEST = "Microsoft.HttpRequest" , SEND_ACTIVITY = "Microsoft.SendActivity" , SWITCH_CONDITION = "Microsoft.SwitchCondition" , TEXT_INPUT = "Microsoft.TextInput" } enum OBIRuleType { INTENT_RULE = "Microsoft.IntentRule" } class ObiDialogNode { readonly dialog : OBITypes . OBIDialog ; intent ? : string ; children : ObiDialogNode [ ] ; requiredCondition ? : OBIUtils . ConditionEntityAndValue ; expectedEntityName ? : string ; constructor ( dialog ) { this . dialog = dialog ; this . children = [ ] ; } } export interface ObiDialogParserResult { luMap : { [ key ] : string [ ] ; } ; lgItems : CLM . LGItem [ ] ; trainDialogs : CLM . TrainDialog [ ] ; warnings : string [ ] ; conditions : { [ key ] : CLM . Condition [ ] ; } ; actionImportIdToExpectedEntityName : { [ key ] : string ; } ; } export class ObiDialogParser { private appId : string ; private actions : CLM . ActionBase [ ] = [ ] ; private entities : CLM . EntityBase [ ] = [ ] ; private dialogs : { [ key ] : OBITypes . OBIDialog ; } ; private lgItems : CLM . LGItem [ ] ; private luMap : { [ key ] : string [ ] ; } ; private warnings : string [ ] ; private conditions : { [ key ] : CLM . Condition [ ] ; } ; private actionImportIdToExpectedEntityName : { [ key ] : string ; } ; private createActionThunkAsync : ( appId : string , action : CLM . ActionBase ) => Promise < CLM . ActionBase | null > ; private createEntityThunkAsync : ( appId : string , entity : CLM . EntityBase ) => Promise < CLM . EntityBase | null > ; constructor ( appId , actions , entities , createActionThunkAsync , createEntityThunkAsync ) { this . appId = appId ; this . actions = [ ... actions ] ; this . entities = [ ... entities ] ; this . createActionThunkAsync = createActionThunkAsync ; this . createEntityThunkAsync = createEntityThunkAsync ; } async parse ( files ) { this . lgItems = [ ] ; this . luMap = { } ; this . dialogs = { } ; this . warnings = [ ] ; this . conditions = { } ; this . actionImportIdToExpectedEntityName = { } ; await this . readDialogFiles ( files ) ; let trainDialogs = [ ] ; const mainDialog = this . dialogs [ "Entry.main" ] ; if ( ! mainDialog ) { this . warnings . push ( `Missing entry point. Expecting a .dialog file called "Entry.main"` ) ; } else { const conditionalEntities = { } ; const rootNode = await this . collectDialogNodes ( mainDialog , conditionalEntities ) ; await this . createConditionalEntities ( conditionalEntities ) ; trainDialogs = await this . getTrainDialogs ( rootNode ) ; } return { luMap : this . luMap , lgItems : this . lgItems , trainDialogs , warnings : this . warnings , conditions : this . conditions , actionImportIdToExpectedEntityName : this . actionImportIdToExpectedEntityName , } ; } private async readDialogFiles ( files ) { for ( const file of files ) { if ( file . name . endsWith ( '.dialog' ) ) { const fileText = await Util . readFileAsync ( file ) ; const obiDialog = JSON . parse ( stripJsonComments ( fileText ) ) ; obiDialog . $id = this . removeSuffix ( fspath . basename ( file . name ) ) ; this . dialogs [ obiDialog . $id ] = obiDialog ; } else if ( file . name . endsWith ( '.lu' ) ) { const fileText = await Util . readFileAsync ( file ) ; this . addToLUMap ( fileText , this . luMap ) ; } else if ( file . name . endsWith ( '.lg' ) ) { const fileText = await Util . readFileAsync ( file ) ; CLM . ObiUtils . addToLGMap ( fileText , this . lgItems ) ; } else { this . warnings . push ( `Expecting .dialog, .lu and .lg files. ${ file . name } is of unknown file type` ) ; } } } private addToLUMap ( text , luMap ) { const keys = text . split ( '##' ) ; for ( const key of keys ) { if ( ! key . startsWith ( ">" ) ) { const inputs = key . split ( '- ' ) . map ( i => i . trim ( ) ) ; luMap [ inputs [ 0 ] ] = inputs . slice ( 1 ) ; } } return luMap ; } /**
     * Walks the dialog tree from the given node.  Validates types of nodes in the tree and returns an
     * in-memory representation of the tree.
     * Tree construction is slightly complicated since child nodes can be referenced in `rules` or `steps`.
     */ private async collectDialogNodes ( obiDialog , conditionalEntities ) { let node = new ObiDialogNode ( obiDialog ) ; if ( obiDialog . rules ) { await this . collectDialogRuleChildren ( node , obiDialog . rules , conditionalEntities ) ; } if ( obiDialog . steps ) { await this . collectDialogStepChildren ( node , obiDialog . steps , conditionalEntities ) ; } return node ; } private async collectDialogRuleChildren ( node , rules , conditionalEntities ) { for ( const rule of rules ) { if ( rule . $type !== OBIRuleType . INTENT_RULE ) { this . warnings . push ( `Unhandled OBI rule type: ${ rule . $type } in ${ node . dialog . $id }` ) ; continue ; } const intent = rule . intent ; if ( ! intent ) { this . warnings . push ( `Rule is missing intent property in ${ node . dialog . $id }` ) ; continue ; } if ( ! rule . steps ) { continue ; } for ( const step of rule . steps ) { if ( typeof step === "string" ) { this . warnings . push ( `Unexpected string step in ${ node . dialog . $id }` ) ; continue ; } if ( step . $type !== OBIStepType . BEGIN_DIALOG || typeof step . dialog !== "string" ) { this . warnings . push ( `Unhandled OBI step type: ${ step . $type } in ${ node . dialog . $id }` ) ; continue ; } const subDialog = this . dialogs [ step . dialog ] ; if ( ! subDialog ) { throw new Error ( `Dialog name ${ step . dialog } undefined` ) ; } const child = await this . collectDialogNodes ( subDialog , conditionalEntities ) ; if ( child ) { child . intent = intent ; node . children . push ( child ) ; } } } } param conditionalEntities param requiredCondition private async collectDialogStepChildren ( node , steps , conditionalEntities , requiredCondition ? ) { for ( const [ i , step ] of steps . entries ( ) ) { if ( typeof step === "string" ) { this . warnings . push ( `Unexpected string step in ${ node . dialog . $id }` ) ; continue ; } switch ( step . $type ) { case OBIStepType . HTTP_REQUEST : { const httpStep = node . dialog . steps ! [ i ] ; const httpDialog = Util . deepCopy ( node . dialog ) ; httpDialog . steps = [ httpStep ] ; const httpNode = new ObiDialogNode ( httpDialog ) ; httpNode . intent = CLM . ModelUtils . generateGUID ( ) ; node . children = [ httpNode ] ; const inputEntityNames = this . getNormalizedHttpInputEntityNames ( step ) ; if ( inputEntityNames ?. size === 1 ) { node . expectedEntityName = inputEntityNames . values ( ) . next ( ) . value ; } const childSteps = Util . deepCopy ( node . dialog . steps ! . slice ( i + 1 ) ) ; const childDialog = Util . deepCopy ( node . dialog ) ; childDialog . steps = childSteps ; const childNode = await this . collectDialogNodes ( childDialog , conditionalEntities ) ; childNode . intent = undefined ; httpNode . children = [ childNode ] ; node . dialog . steps = node . dialog . steps ! . slice ( 0 , i ) ; return ; } case OBIStepType . BEGIN_DIALOG : { if ( ! step . dialog || typeof step . dialog !== "string" ) { this . warnings . push ( `Invalid dialog in ${ node . dialog . $id }` ) ; continue ; } const subDialog = this . dialogs [ step . dialog ] ; if ( ! subDialog ) { throw new Error ( `Dialog name ${ step . dialog } undefined` ) ; } const childDialog = await this . collectDialogNodes ( subDialog , conditionalEntities ) ; if ( childDialog ) { if ( requiredCondition ) { childDialog . requiredCondition = requiredCondition ; } node . children . push ( childDialog ) ; } break ; } case OBIStepType . SWITCH_CONDITION : if ( ! step . cases && ! step . default ) { throw new Error ( "SwitchCondition must have at least one case or default" ) ; } if ( step . cases ) { for ( const branch of step . cases ) { if ( ! branch . steps ) { throw new Error ( "Each case in SwitchCondition must have at least one step" ) ; } const requiredConditionFromCase = OBIUtils . parseEntityConditionFromDialogCase ( branch , conditionalEntities ) ; await this . collectDialogStepChildren ( node , branch . steps , conditionalEntities , requiredConditionFromCase ) ; } } if ( step . default ) { await this . collectDialogStepChildren ( node , step . default , conditionalEntities ) ; } break ; default : } } } private async getTrainDialogs ( node ) { return this . getTrainDialogsIter ( node , [ ] , node . intent ) ; } param node param currentRounds param intent param requiredCondition private async getTrainDialogsIter ( node , currentRounds , intent , requiredCondition ? ) { if ( ! node ) { return [ ] ; } let currentIntent = intent ; if ( currentIntent ) { if ( node . intent && node . intent !== currentIntent ) { throw new Error ( `Node intent ${ node . intent } conflicts with incoming intent ${ currentIntent }` ) ; } } else { currentIntent = node . intent ; } let currentRequiredCondition = requiredCondition ; if ( currentRequiredCondition ) { if ( node . requiredCondition && node . requiredCondition !== currentRequiredCondition ) { const nodeRequiredConditionJson = JSON . stringify ( node . requiredCondition ) ; const currRequiredConditionJson = JSON . stringify ( currentRequiredCondition ) ; throw new Error ( `Node required entity ${ nodeRequiredConditionJson } conflict with incoming required entity ${ currRequiredConditionJson }` ) ; } } else { currentRequiredCondition = node . requiredCondition ; } let rounds = Util . deepCopy ( currentRounds ) ; const obiDialog = node . dialog ; if ( obiDialog . steps ) { let scorerSteps = await this . getScorerStepsFromOBIDialogSteps ( obiDialog . steps ) ; if ( scorerSteps . length > 0 ) { if ( currentRequiredCondition ) { const condition = this . getConditionFromRequiredEntity ( currentRequiredCondition ) ; if ( ! scorerSteps [ 0 ] . importId ) { scorerSteps [ 0 ] . importId = CLM . ModelUtils . generateGUID ( ) ; } if ( ! this . conditions [ scorerSteps [ 0 ] . importId ] ) { this . conditions [ scorerSteps [ 0 ] . importId ] = [ ] ; } this . conditions [ scorerSteps [ 0 ] . importId ] . push ( condition ) ; currentRequiredCondition = undefined ; } if ( node . expectedEntityName ) { const lastStepIndex = scorerSteps . length - 1 ; if ( ! scorerSteps [ lastStepIndex ] . importId ) { scorerSteps [ lastStepIndex ] . importId = CLM . ModelUtils . generateGUID ( ) ; } this . actionImportIdToExpectedEntityName [ scorerSteps [ lastStepIndex ] . importId ! ] = node . expectedEntityName ; } if ( currentIntent ) { const extractorStep = { textVariations : this . getTextVariations ( currentIntent ) } ; const trainRound = { extractorStep , scorerSteps } ; currentIntent = undefined ; rounds . push ( trainRound ) ; } else { if ( rounds . length === 0 ) { throw new Error ( `Attempting to append scorer steps to a non-existent round in node ${ obiDialog . $id }` ) ; } let round = rounds [ rounds . length - 1 ] ; round . scorerSteps = [ ... round . scorerSteps , ... scorerSteps ] ; } } } if ( ! node . children || node . children . length === 0 ) { let dialog = this . makeEmptyTrainDialog ( ) ; dialog . rounds = [ ... rounds ] ; return [ dialog ] ; } let dialogs = [ ] ; for ( const child of node . children ) { dialogs = [ ... dialogs , ... ( await this . getTrainDialogsIter ( child , rounds , currentIntent , currentRequiredCondition ) ) ] ; } return dialogs ; } private async getScorerStepsFromOBIDialogSteps ( steps ) { const scorerSteps = [ ] ; for ( const [ i , step ] of steps . entries ( ) ) { const nextStep = ( i + 1 < steps . length ) ? steps [ i + 1 ] : undefined ; if ( typeof step === "string" || typeof nextStep === "string" ) { this . warnings . push ( `Unexpected string step` ) ; continue ; } switch ( step . $type ) { case OBIStepType . SEND_ACTIVITY : { if ( ! step . activity ) { throw new Error ( "Expected activity to be set in steps" ) ; } const scorerStep = await this . getScorerStepFromActivity ( step . activity ) ; scorerSteps . push ( scorerStep ) ; break ; } case OBIStepType . TEXT_INPUT : { if ( ! step . prompt ) { throw new Error ( "Expected activity to be set in steps" ) ; } const scorerStep = await this . getScorerStepFromActivity ( step . prompt ) ; scorerSteps . push ( scorerStep ) ; break ; } case OBIStepType . HTTP_REQUEST : { const scorerStep = await this . createActionAndEntitiesFromHttpRequest ( step , nextStep ) ; scorerSteps . push ( scorerStep ) ; break ; } case OBIStepType . SWITCH_CONDITION : { let childSteps = [ ] ; if ( step . cases ) { for ( const branch of step . cases ) { if ( ! branch . steps ) { throw new Error ( "Case branch must contain steps" ) ; } childSteps = [ ... childSteps , ... await this . getScorerStepsFromOBIDialogSteps ( branch . steps ) ] ; } } if ( step . default ) { childSteps = [ ... childSteps , ... await this . getScorerStepsFromOBIDialogSteps ( step . default ) ] ; } if ( childSteps . length > 0 ) { throw new Error ( "SwitchConditions containing action steps are not currently supported" ) ; } if ( nextStep ) { const remainingSteps = steps . slice ( i + 1 ) ; childSteps = await this . getScorerStepsFromOBIDialogSteps ( remainingSteps ) ; if ( childSteps . length > 0 ) { throw new Error ( "SwitchCondition may not be followed by an action step in the same node" ) ; } } return scorerSteps ; } case OBIStepType . BEGIN_DIALOG : { break ; } case OBIStepType . END_DIALOG : case OBIStepType . END_TURN : break ; default : { this . warnings . push ( `Unhandled OBI Type: ${ step . $type }` ) ; } } } return scorerSteps ; } /**
     * If the step is an HttpRequest node that has input variables, returns a `Set` containing the normalized
     * name of those variables.  Otherwise returns undefined.
     */ private getNormalizedHttpInputEntityNames ( step ) { if ( step ?. $type !== OBIStepType . HTTP_REQUEST ) { return undefined ; } if ( ! step ?. body ) { return undefined ; } const entityNames = new Set ( ) ; for ( const entityName of Object . keys ( step . body ) ) { const normalizedEntityName = this . normalizeValueName ( entityName , CLM . MAX_ENTITY_NAME_LENGTH ) ; entityNames . add ( normalizedEntityName ) ; } return entityNames ; } param conditionalEntities private async createConditionalEntities ( conditionalEntities ) { for ( const entityName of Object . keys ( conditionalEntities ) ) { let foundEntity = this . entities . find ( e => e . entityName === entityName ) ; if ( foundEntity ) { throw new Error ( `Unexpected: multiple definitions for ${ entityName }` ) ; } await this . createEnumEntity ( entityName , conditionalEntities [ entityName ] ) ; } } /**
     * Creates a new enum entity with `values`.  Returns the new enum entity if successful.
     * Note that entity values will be truncated to the max length allowed by the backend.
     * Throws an error if multiple distinct condition names have the same truncated value.
     */ private async createEnumEntity ( entityName , values ) { let enumValues = [ ] ; let updatedValues = { } ; for ( const value of values ) { const normalized = this . normalizeValueName ( value , CLM . MAX_ENUM_VALUE_NAME_LENGTH ) ; if ( ! updatedValues [ normalized ] ) { updatedValues [ normalized ] = new Set ( [ value ] ) ; } else { const fullValues = updatedValues [ normalized ] ; if ( ! fullValues . has ( value ) ) { const existing = fullValues . values ( ) . next ( ) . value ; throw new Error ( `Can't create enum, values ${ value } and ${ existing } map to the same normalized string` ) ; } } enumValues . push ( { enumValue : normalized } ) ; } const newEntity = { entityId : undefined ! , entityName , resolverType : NONE_RESOLVER_KEY , createdDateTime : new Date ( ) . toJSON ( ) , lastModifiedDateTime : new Date ( ) . toJSON ( ) , isResolutionRequired : false , isMultivalue : false , isNegatible : false , negativeId : null , positiveId : null , entityType : CLM . EntityType . ENUM , enumValues , version : null , packageCreationId : null , packageDeletionId : null , doNotMemorize : false } ; const entityId = await ( ( this . createEntityThunkAsync ( this . appId , newEntity ) as any ) as Promise < string > ) ; if ( ! entityId ) { throw new Error ( `Failed to create entity ${ entityName }` ) ; } newEntity . entityId = entityId ; this . entities . push ( newEntity ) ; return newEntity ; } private async createCustomTrainedEntity ( entityName ) { const newEntity = { entityId : undefined ! , entityName , resolverType : NONE_RESOLVER_KEY , createdDateTime : new Date ( ) . toJSON ( ) , lastModifiedDateTime : new Date ( ) . toJSON ( ) , isResolutionRequired : false , isMultivalue : false , isNegatible : false , negativeId : null , positiveId : null , entityType : CLM . EntityType . LUIS , enumValues : undefined , version : null , packageCreationId : null , packageDeletionId : null , doNotMemorize : false } ; const entityId = await ( ( this . createEntityThunkAsync ( this . appId , newEntity ) as any ) as Promise < string > ) ; if ( ! entityId ) { throw new Error ( `Failed to create entity ${ entityName }` ) ; } newEntity . entityId = entityId ; this . entities . push ( newEntity ) ; return newEntity ; } /**
     * Returns a `Condition` referencing an enum entity and enum value id, which is built from the entity and value name.
     */ private getConditionFromRequiredEntity ( requiredEntity ) { const conditionEntity = this . entities . find ( e => e . entityName === requiredEntity . entity ) ; if ( ! conditionEntity ) { throw new Error ( `Couldn't find entity ${ requiredEntity . entity }` ) ; } if ( conditionEntity . entityType !== CLM . EntityType . ENUM || ! conditionEntity . enumValues ) { throw new Error ( `Entity ${ conditionEntity . entityName } is not a valid enum` ) ; } const normalizedValueName = this . normalizeValueName ( requiredEntity . value , CLM . MAX_ENUM_VALUE_NAME_LENGTH ) ; const enumValueId = conditionEntity . enumValues . find ( v => v . enumValue === normalizedValueName ) ; if ( ! enumValueId ) { throw new Error ( `Couldn't find value ${ normalizedValueName } on enum entity ${ conditionEntity . entityName }` ) ; } return { entityId : conditionEntity . entityId , valueId : enumValueId . enumValueId , condition : CLM . ConditionType . EQUAL } ; } private async getScorerStepFromActivity ( prompt ) { let scoreInput = { filledEntities : [ ] , context : { } , maskedActions : [ ] } ; return { importText : prompt , input : scoreInput , labelAction : CLM . CL_STUB_IMPORT_ACTION_ID , logicResult : undefined , scoredAction : undefined } ; } /**
     * HttpRequest represents a RESTful request with known input and output parameters.
     * The .dialog file is expected to have a field `responseFields` that enumerates the top-level
     * output parameters of the response object.  Note that as of 2019.09, this field is specific
     * to ConversationLearner and is not part of the OBI spec.
     */ private async createActionAndEntitiesFromHttpRequest ( step , nextStep ) { if ( ! step . url ) { throw new Error ( 'HTTP requests require url' ) ; } if ( ! step . body ) { throw new Error ( 'HTTP requests require body' ) ; } const apiInputEntityNames = this . getNormalizedHttpInputEntityNames ( step ) ; if ( apiInputEntityNames ) { for ( const entityName of apiInputEntityNames ) { let entity = this . entities . find ( e => e . entityName === entityName ) ; if ( entity ) { continue ; } await this . createCustomTrainedEntity ( entityName ) ; } } const isTerminal = ( ! nextStep || nextStep . $type === OBIStepType . TEXT_INPUT || nextStep . $type === OBIStepType . END_TURN ) ; const hashText = JSON . stringify ( step ) ; let action = OBIUtils . findActionFromHashText ( hashText , this . actions ) ; if ( ! action && this . createActionThunkAsync ) { action = await DialogEditing . getOrCreatePlaceholderAPIAction ( this . appId , step . url , isTerminal , this . actions , this . createActionThunkAsync as any ) ; } let actionOutputEntities = [ ] ; if ( step . responseFields ) { actionOutputEntities = step . responseFields . map ( ( field ) => { return { entityName : field } ; } ) ; } const filledEntities = await OBIUtils . importActionOutput ( actionOutputEntities , this . entities , this . appId , this . createEntityThunkAsync ) ; const scoreInput = { filledEntities , context : { } , maskedActions : [ ] } ; let scoredAction ; if ( action ) { scoredAction = { actionId : action . actionId , payload : action . payload , isTerminal : action . isTerminal , actionType : CLM . ActionTypes . API_LOCAL , score : 1 } ; } return { importText : undefined , input : scoreInput , labelAction : action ? action . actionId : CLM . CL_STUB_IMPORT_ACTION_ID , logicResult : undefined , scoredAction } ; } private getTextVariations ( intentName ) { let userInputs = this . luMap [ intentName ] ; if ( ! userInputs || userInputs . length === 0 ) { userInputs = [ intentName ] ; } userInputs = userInputs . slice ( 0 , CLM . MAX_TEXT_VARIATIONS ) ; const textVariations = [ ] ; userInputs . forEach ( input => { textVariations . push ( { text : input , labelEntities : [ ] } ) ; } ) ; return textVariations ; } private normalizeValueName ( name , maxLength ) { return name . substr ( 0 , maxLength ) ; } private makeEmptyTrainDialog ( ) { return { trainDialogId : undefined ! , version : undefined ! , packageCreationId : undefined ! , packageDeletionId : undefined ! , sourceLogDialogId : undefined ! , initialFilledEntities : [ ] , rounds : [ ] , tags : [ ] , description : '' , createdDateTime : new Date ( ) . toJSON ( ) , lastModifiedDateTime : new Date ( ) . toJSON ( ) , validity : CLM . Validity . INVALID , } ; } private removeSuffix ( text ) { let name = text . split ( '.' ) ; name . pop ( ) ; return name . join ( '.' ) ; } }