/**
 * Copyright (c) Microsoft Corporation. All rights reserved.
 * Licensed under the MIT License.
 */ import { Value } from 'slate' ; import * as models from './models' ; import * as util from '../../Utils/util' ; import * as CLM from '@conversationlearner/models' ; /**
 * Recursively walk up DOM tree until root or parent with non-static position is found.
 * (relative, fixed, or absolute) which will be used as reference for absolutely positioned elements within it
 */ export const getRelativeParent = ( element ) : HTMLElement => { if ( ! element ) { return document . body ; } const position = window . getComputedStyle ( element ) . getPropertyValue ( 'position' ) ; if ( position !== 'static' ) { return element ; } return getRelativeParent ( element . parentElement ) ; } ; export const getSelectedText = ( value ) => { const characters = value . characters ? value . characters . toJSON ( ) : [ ] ; return characters . reduce ( ( s , node ) => s + node . text , '' ) ; } ; export const valueToJSON = ( value ) => { return { data : value . data . toJSON ( ) , decorations : value . decorations ? value . decorations . toJSON ( ) : [ ] , document : value . toJSON ( ) . document , activeMarks : value . activeMarks . toJSON ( ) , marks : value . marks . toJSON ( ) , texts : value . texts . toJSON ( ) , selectedText : getSelectedText ( value ) , selection : value . selection . toJSON ( ) } ; } ; export interface IToken { text : string ; isSelectable : boolean ; startIndex : number ; endIndex : number ; } interface ICustomEntityWithTokenIndices extends models . IGenericEntity < any > { startTokenIndex : number ; endTokenIndex : number ; } interface IEntityPlaceholder { entity : models . IGenericEntity < any > ; tokens : IToken [ ] ; } export type TokenArray = ( IToken | IEntityPlaceholder ) [ ] ; export const tokenizeText = ( text , tokenRegex ) : IToken [ ] => { const tokens = [ ] ; if ( text . length === 0 ) { return tokens ; } let result = null ; let lastIndex = tokenRegex . lastIndex ; while ( ( result = tokenRegex . exec ( text ) ) !== null ) { const matchedText = text . substring ( lastIndex , result . index ) ; tokens . push ( ... [ { text : matchedText , isSelectable : true , startIndex : lastIndex , endIndex : result . index } , { text : result [ 0 ] , isSelectable : false , startIndex : result . index , endIndex : result . index + result [ 0 ] . length } ] ) ; lastIndex = tokenRegex . lastIndex ; } const endIndex = text . length ; tokens . push ( { text : text . substring ( lastIndex , endIndex ) , isSelectable : true , startIndex : lastIndex , endIndex } ) ; return tokens ; } ; param xs param f export const findLastIndex = < T > ( xs , f ) : number => { for ( let i = xs . length - 1 ; i >= 0 ; i -- ) { if ( f ( xs [ i ] ) ) { return i ; } } return - 1 ; } ; param tokens param customEntities export const addTokenIndicesToCustomEntities = ( tokens , customEntities ) : ICustomEntityWithTokenIndices [ ] => { return customEntities . map < ICustomEntityWithTokenIndices > ( ce => { const startTokenIndex = tokens . findIndex ( t => t . isSelectable === true && ce . startIndex < t . endIndex && t . endIndex <= ce . endIndex ) ; const endTokenIndex = findLastIndex ( tokens , t => t . isSelectable === true && ce . startIndex <= t . startIndex && t . startIndex < ce . endIndex ) ; if ( startTokenIndex === - 1 || endTokenIndex === - 1 ) { console . warn ( `Could not find valid token for custom entity: ` , ce ) ; } return { ... ce , startTokenIndex , endTokenIndex : endTokenIndex + 1 } ; } ) ; } ; param tokens param customEntitiesWithTokens export const wrapTokensWithEntities = ( tokens , customEntitiesWithTokens ) : TokenArray => { if ( customEntitiesWithTokens . length === 0 ) { return tokens ; } const sortedCustomEntities = [ ... customEntitiesWithTokens ] . sort ( ( a , b ) => a . startIndex - b . startIndex ) ; const firstCet = sortedCustomEntities [ 0 ] ; const tokenArray = [ ... tokens . slice ( 0 , firstCet . startTokenIndex ) ] ; for ( const [ i , cet ] of Array . from ( sortedCustomEntities . entries ( ) ) ) { tokenArray . push ( { entity : cet , tokens : tokens . slice ( cet . startTokenIndex , cet . endTokenIndex ) } ) ; if ( i !== sortedCustomEntities . length - 1 ) { const nextCet = sortedCustomEntities [ i + 1 ] ; tokenArray . push ( ... tokens . slice ( cet . endTokenIndex , nextCet . startTokenIndex ) ) ; } } const lastCet = sortedCustomEntities [ sortedCustomEntities . length - 1 ] ; tokenArray . push ( ... tokens . slice ( lastCet . endTokenIndex ) ) ; return tokenArray ; } ; export const labelTokens = ( tokens , customEntities ) : TokenArray => { return wrapTokensWithEntities ( tokens , addTokenIndicesToCustomEntities ( tokens , customEntities ) ) ; } ; export const convertToSlateNodes = ( tokensWithEntities , inlineNodeType = models . NodeType . CustomEntityNodeType ) : any [ ] => { const nodes = [ ] ; if ( tokensWithEntities . length === 0 ) { nodes . push ( { "kind" : "text" , "leaves" : [ { "kind" : "leaf" , "text" : '' , "marks" : [ ] } ] } ) ; return nodes ; } for ( const tokenOrEntity of tokensWithEntities ) { if ( ( tokenOrEntity as IEntityPlaceholder ) . entity ) { const entityPlaceholder = tokenOrEntity as any ; const nestedNodes = convertToSlateNodes ( entityPlaceholder . tokens , inlineNodeType ) ; nodes . push ( { "kind" : "inline" , "type" : inlineNodeType , "isVoid" : false , "data" : entityPlaceholder . entity . data , "nodes" : nestedNodes } ) ; } else { const token = tokenOrEntity as any ; if ( token . isSelectable ) { nodes . push ( { "kind" : "inline" , "type" : models . NodeType . TokenNodeType , "isVoid" : false , "data" : token , "nodes" : [ { "kind" : "text" , "leaves" : [ { "kind" : "leaf" , "text" : token . text , "marks" : [ ] } ] } ] } ) ; } else { nodes . push ( { "kind" : "text" , "leaves" : [ { "kind" : "leaf" , "text" : token . text , "marks" : [ ] } ] } ) ; } } } return nodes ; } ; export const convertToSlateValue = ( tokensWithEntities , inlineNodeType = models . NodeType . CustomEntityNodeType ) : any => { const nodes = convertToSlateNodes ( tokensWithEntities , inlineNodeType ) ; const document = { "document" : { "nodes" : [ { "kind" : "block" , "type" : "paragraph" , "isVoid" : false , "data" : { } , "nodes" : nodes } ] } } ; return Value . fromJSON ( document ) ; } ; function convertSegmentsToSlateValue ( normalizedSegements , inlineType = models . NodeType . CustomEntityNodeType ) { const nodes = normalizedSegements . map ( segement => { if ( segement . type === 'inline' ) { return { "kind" : "inline" , "type" : inlineType , "isVoid" : false , "data" : segement . data , "nodes" : [ { "kind" : "text" , "leaves" : [ { "kind" : "leaf" , "text" : segement . text , "marks" : [ ] } ] } ] } ; } return { "kind" : "text" , "leaves" : [ { "kind" : "leaf" , "text" : segement . text , "marks" : [ ] } ] } ; } ) ; return { "document" : { "nodes" : [ { "kind" : "block" , "type" : "paragraph" , "isVoid" : false , "data" : { } , "nodes" : nodes } ] } } ; } export const discardOverlappingEntities = ( customEntities ) : models . IGenericEntity < any > [ ] => { const entityGroups = customEntities . reduce < models . IGenericEntity < object > [ ] [ ] > ( ( groups , customEntity ) => { const existingGroup = groups . find ( g => g . some ( ce => ce . startIndex === customEntity . startIndex ) ) ; if ( existingGroup ) { existingGroup . push ( customEntity ) ; } else { const newGroup = [ customEntity ] ; groups . push ( newGroup ) ; } return groups ; } , [ ] ) ; const sortedEntityGroups = entityGroups . map ( g => [ ... g ] . sort ( ( a , b ) => { const sizeA = a . endIndex - a . startIndex ; const sizeB = b . endIndex - b . startIndex ; return sizeA - sizeB ; } ) ) ; let lastEndIndex = 0 ; const nonOverlappingEntities = [ ] ; for ( const group of sortedEntityGroups ) { const firstEntityOfGroup = group [ 0 ] ; if ( ! firstEntityOfGroup ) { throw new Error ( `Error when discarding overlapping entities. Cannot get first entity of group because the group is empty.` ) ; } if ( firstEntityOfGroup . startIndex >= lastEndIndex ) { nonOverlappingEntities . push ( firstEntityOfGroup ) ; lastEndIndex = firstEntityOfGroup . endIndex ; } } return nonOverlappingEntities ; } ; param customEntities export const warnAboutOverlappingEntities = ( customEntities ) : boolean => { return customEntities . some ( ( entity , i ) => { return customEntities . slice ( i + 1 ) . some ( ( otherEntity , _ , es ) => { const overlapStartIndex = ( otherEntity . startIndex <= entity . startIndex && otherEntity . endIndex >= entity . startIndex ) ; const overlapEndIndex = ( otherEntity . startIndex <= entity . endIndex && otherEntity . endIndex >= entity . endIndex ) ; const overlap = overlapStartIndex || overlapEndIndex ; if ( overlap ) { console . warn ( `Custom entities have overlap. Overlapping entities will be discarded to allow proper rendering in UI but this is a bug.` , customEntities ) ; } return overlap ; } ) ; } ) ; } ; /**
 * Note: this is more like a negative match used to determine characters that split the string instead of
 * positive match would specify characters which are tokens. Only chose this because it seems like a much
 * simpler regex / smaller set of characters, but I imagine alternative approach would work
 */ export const tokenizeRegex = /\s+|[.?,!]/g ; param text param customEntities export const convertEntitiesAndTextToTokenizedEditorValue = ( text , customEntities , inlineNodeType ) => { warnAboutOverlappingEntities ( customEntities ) ; const nonOverlappingCustomEntities = discardOverlappingEntities ( customEntities ) ; const labeledTokens = labelTokens ( tokenizeText ( text , tokenizeRegex ) , nonOverlappingCustomEntities ) ; return convertToSlateValue ( labeledTokens , inlineNodeType ) ; } ; param text param customEntities param inlineNodeType export const convertEntitiesAndTextToEditorValue = ( text , customEntities , inlineNodeType ) => { const initialSegment = { text , startIndex : 0 , endIndex : text . length , type : models . SegementType . Normal , data : { } } ; const normalizedSegements = customEntities . reduce < models . ISegement [ ] > ( ( segements , entity ) => { const segementIndexWhereEntityBelongs = segements . findIndex ( seg => seg . startIndex <= entity . startIndex && entity . endIndex <= seg . endIndex ) ; if ( segementIndexWhereEntityBelongs === - 1 ) { throw new Error ( `When attempting to convert entities to editor value, could not find text segement to place entity. Entity indices are: [${ entity . startIndex }, ${ entity . endIndex }] but available segement ranges are: ${ segements . map ( s => `[${ s . startIndex }, ${ s . endIndex }]` ) . join ( `, ` ) }` ) ; } const prevSegements = segements . slice ( 0 , segementIndexWhereEntityBelongs ) ; const nextSegements = segements . slice ( segementIndexWhereEntityBelongs + 1 , segements . length ) ; const segementWhereEntityBelongs = segements [ segementIndexWhereEntityBelongs ] ; const absolutePrevSegementEndIndex = entity . startIndex ; const relativePrevSegementEndIndex = entity . startIndex - segementWhereEntityBelongs . startIndex ; const prevSegementText = segementWhereEntityBelongs . text . substring ( 0 , relativePrevSegementEndIndex ) ; const prevSegement = { ... segementWhereEntityBelongs , text : prevSegementText , endIndex : absolutePrevSegementEndIndex , } ; const absoluteNextSegementStartIndex = entity . endIndex ; const relativeNextSegementStartIndex = entity . endIndex - segementWhereEntityBelongs . startIndex ; const nextSegementText = segementWhereEntityBelongs . text . substring ( relativeNextSegementStartIndex ) ; const nextSegement = { ... segementWhereEntityBelongs , text : nextSegementText , startIndex : absoluteNextSegementStartIndex , } ; const newSegement = { text : segementWhereEntityBelongs . text . substring ( relativePrevSegementEndIndex , relativeNextSegementStartIndex ) , startIndex : entity . startIndex , endIndex : entity . endIndex , type : models . SegementType . Inline , data : entity . data } ; const newSegements = prevSegements ; if ( prevSegement . startIndex !== prevSegement . endIndex ) { newSegements . push ( prevSegement ) ; } if ( newSegement . startIndex !== newSegement . endIndex ) { newSegements . push ( newSegement ) ; } if ( nextSegement . startIndex !== nextSegement . endIndex ) { newSegements . push ( nextSegement ) ; } return [ ... newSegements , ... nextSegements ] ; } , [ initialSegment ] ) ; const document = convertSegmentsToSlateValue ( normalizedSegements , inlineNodeType ) ; return Value . fromJSON ( document ) ; } ; param change export const getEntitiesFromValueUsingTokenData = ( change ) : models . IGenericEntity < models . IGenericEntityData < CLM . PredictedEntity > > [ ] => { const entityInlineNodes = change . value . document . filterDescendants ( ( node ) => node . type === models . NodeType . CustomEntityNodeType ) ; return ( entityInlineNodes . map ( ( entityNode ) => { const tokenInlineNodes = entityNode . filterDescendants ( ( node ) => node . type === models . NodeType . TokenNodeType ) . toJS ( ) ; if ( tokenInlineNodes . length === 0 ) { console . warn ( `Error 'getEntitiesFromValue': found entity node which did not contain any token nodes ` ) ; return null ; } const firstToken = tokenInlineNodes [ 0 ] . data ; const lastToken = tokenInlineNodes [ tokenInlineNodes . length - 1 ] . data ; const data = entityNode . data . toJS ( ) ; return { startIndex : firstToken . startIndex , endIndex : lastToken . endIndex , data } ; } ) . toJS ( ) as any [ ] ) . filter ( x => x ) ; } ; export const getPreBuiltEntityDisplayName = ( entity , pe ) : string => { if ( typeof pe . builtinType !== 'string' || pe . builtinType . length === 0 ) { return entity . entityType ; } const names = pe . builtinType . split ( '.' ) ; const [ builtinPrefix , ... segements ] = names ; /**
     * Expect resolutions to have builtinType to be of form:
     * builtin.datetimeV2.duration
     * If it somehow is not '.'(period) delimeted. Return whole type
     */ if ( segements . length === 0 ) { return builtinPrefix ; } return segements . join ( '.' ) ; } ; export const convertPredictedEntityToGenericEntity = ( pe , entityName , displayName ) : models . IGenericEntity < models . IGenericEntityData < CLM . PredictedEntity > > => ( { startIndex : pe . startCharIndex , endIndex : pe . startCharIndex + pe . entityText . length , data : { option : { id : pe . entityId , name : entityName , type : pe . builtinType , resolverType : null } , text : pe . entityText , displayName , original : pe , } } ) ; export const convertGenericEntityToPredictedEntity = ( entities ) => ( ge ) : CLM . PredictedEntity => { const predictedEntity = ge . data . original ; if ( predictedEntity ) { return predictedEntity ; } const option = ge . data . option ; const text = ge . data . text ?? '' ; if ( option . type !== CLM . EntityType . LUIS ) { console . warn ( `convertGenericEntityToPredictedEntity option selected as option type other than LUIS, this will most likely cause an error` ) ; } const entity = entities . find ( e => e . entityId === option . id ) ; if ( ! entity ) { throw new Error ( `Could not find entity with id: ${ option . id } in list of entities: ${ entities }` ) ; } if ( text . length !== ( ge . endIndex - ge . startIndex ) ) { throw new Error ( `Entity labeling error. Start and End index difference don't match the length of selected text. This shouldn't be possible.` ) ; } return { entityId : entity . entityId , startCharIndex : ge . startIndex , endCharIndex : ge . endIndex - 1 , entityText : text , resolution : { } , builtinType : entity . entityType , score : 0 } ; } ; export const convertExtractorResponseToEditorModels = ( extractResponse , entities ) : models . IEditorProps => { const options = entities . filter ( e => e . entityType === CLM . EntityType . LUIS ) . map < models . IOption > ( e => ( { id : e . entityId , name : util . entityDisplayName ( e ) , type : e . entityType , resolverType : e . resolverType } ) ) ; const text = extractResponse . text ; const internalPredictedEntities = extractResponse . predictedEntities . map ( predictedEntity => { const entity = entities . find ( e => e . entityId === predictedEntity . entityId ) ; return { entity , predictedEntity } ; } ) . filter ( ipe => ipe . entity !== null ) ; const customEntities = internalPredictedEntities . filter ( ( { entity } ) => entity ?. entityType === CLM . EntityType . LUIS ) . map ( ( { entity , predictedEntity } ) => convertPredictedEntityToGenericEntity ( predictedEntity , entity ! . entityName , util . entityDisplayName ( entity ! ) ) ) ; const preBuiltEntities = internalPredictedEntities . filter ( ( { entity } ) => entity && CLM . isPrebuilt ( entity ) ) . map ( ( { entity , predictedEntity } ) => convertPredictedEntityToGenericEntity ( predictedEntity , entity ! . entityName , getPreBuiltEntityDisplayName ( entity ! , predictedEntity ) ) ) ; return { options , text , customEntities , preBuiltEntities } ; } ;