import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey' ; import { IScopedClusterClient } from 'kibana/server' ; import { chunk } from 'lodash' ; import { CATEGORY_EXAMPLES_SAMPLE_SIZE } from '../../../../../common/constants/categorization_job' ; import { Token , CategorizationAnalyzer , CategoryFieldExample , } from '../../../../../common/types/categories' ; import { RuntimeMappings } from '../../../../../common/types/fields' ; import { IndicesOptions } from '../../../../../common/types/anomaly_detection_jobs' ; import { ValidationResults } from './validation_results' ; const CHUNK_SIZE = 100 ; export function categorizationExamplesProvider ( { asCurrentUser , asInternalUser , } ) { const validationResults = new ValidationResults ( ) ; async function categorizationExamples ( indexPatternTitle , query , size , categorizationFieldName , timeField , start , end , analyzer , runtimeMappings , indicesOptions ) { if ( timeField !== undefined ) { const range = { range : { [ timeField ] : { gte : start , lt : end , format : 'epoch_millis' , } , } , } ; if ( query . bool === undefined ) { query . bool = { } ; } if ( query . bool . filter === undefined ) { query . bool . filter = range ; } else { if ( Array . isArray ( query . bool . filter ) ) { query . bool . filter . push ( range ) ; } else { query . bool . filter . range = range ; } } } const { body } = await asCurrentUser . search < estypes . SearchResponse < { [ id ] : string ; } > > ( { index : indexPatternTitle , size , body : { fields : [ categorizationFieldName ] , _source : false , query , sort : [ '_doc' ] , ... ( runtimeMappings !== undefined ? { runtime_mappings : runtimeMappings } : { } ) , } , ... ( indicesOptions ?? { } ) , } ) ; const tempExamples = body . hits . hits . map ( ( { fields } ) => fields && Array . isArray ( fields [ categorizationFieldName ] ) && fields [ categorizationFieldName ] . length > 0 ? fields [ categorizationFieldName ] [ 0 ] : null ) ; validationResults . createNullValueResult ( tempExamples ) ; const allExamples = tempExamples . filter ( ( example ) => example !== undefined && example !== null ) ; validationResults . createMedianMessageLengthResult ( allExamples ) ; try { const examplesWithTokens = await getTokens ( CHUNK_SIZE , allExamples , analyzer ) ; return { examples : examplesWithTokens } ; } catch ( err ) { const halfExamples = allExamples . splice ( 0 , Math . ceil ( allExamples . length / 2 ) ) ; const halfChunkSize = CHUNK_SIZE / 2 ; try { const examplesWithTokens = await getTokens ( halfChunkSize , halfExamples , analyzer ) ; return { examples : examplesWithTokens } ; } catch ( error ) { validationResults . createTooManyTokensResult ( error , halfChunkSize ) ; return { examples : halfExamples . map ( ( e ) => ( { text : e , tokens : [ ] } ) ) } ; } } } async function getTokens ( chunkSize , examples , analyzer ) { const exampleChunks = chunk ( examples , chunkSize ) ; const tokensPerExampleChunks = [ ] ; for ( const c of exampleChunks ) { tokensPerExampleChunks . push ( await loadTokens ( c , analyzer ) ) ; } const tokensPerExample = tokensPerExampleChunks . flat ( ) ; return examples . map ( ( e , i ) => ( { text : e , tokens : tokensPerExample [ i ] } ) ) ; } async function loadTokens ( examples , analyzer ) { const { body : { tokens } , } = await asInternalUser . indices . analyze ( { body : { ... getAnalyzer ( analyzer ) , text : examples , } , } ) ; const lengths = examples . map ( ( e ) => e . length ) ; const sumLengths = lengths . map ( ( ( s ) => ( a ) => ( s += a ) ) ( 0 ) ) ; const tokensPerExample = examples . map ( ( e ) => [ ] ) ; if ( tokens !== undefined ) { tokens . forEach ( ( t , i ) => { for ( let g = 0 ; g < sumLengths . length ; g ++ ) { if ( t . start_offset <= sumLengths [ g ] + g ) { const offset = g > 0 ? sumLengths [ g - 1 ] + g : 0 ; const start = t . start_offset - offset ; tokensPerExample [ g ] . push ( { ... t , start_offset : start , end_offset : start + t . token . length , } ) ; break ; } } } ) ; } return tokensPerExample ; } function getAnalyzer ( analyzer ) { if ( typeof analyzer === 'object' && analyzer . tokenizer !== undefined ) { return analyzer ; } else { return { analyzer : 'standard' } ; } } async function validateCategoryExamples ( indexPatternTitle , query , size , categorizationFieldName , timeField , start , end , analyzer , runtimeMappings , indicesOptions ) { const resp = await categorizationExamples ( indexPatternTitle , query , CATEGORY_EXAMPLES_SAMPLE_SIZE , categorizationFieldName , timeField , start , end , analyzer , runtimeMappings , indicesOptions ) ; const { examples } = resp ; const sampleSize = examples . length ; validationResults . createTokenCountResult ( examples , sampleSize ) ; const sortedExamples = examples . map ( ( e , i ) => ( { ... e , origIndex : i } ) ) . sort ( ( a , b ) => b . tokens . length - a . tokens . length ) ; const multiple = Math . floor ( sampleSize / size ) || sampleSize ; const filteredExamples = [ ] ; let i = 0 ; while ( filteredExamples . length < size && i < sampleSize ) { filteredExamples . push ( sortedExamples [ i ] ) ; i += multiple ; } const processedExamples = filteredExamples . sort ( ( a , b ) => a . origIndex - b . origIndex ) . map ( ( e ) => ( { text : e . text , tokens : e . tokens } ) ) ; return { overallValidStatus : validationResults . overallResult , validationChecks : validationResults . results , sampleSize , examples : processedExamples , } ; } return { validateCategoryExamples , } ; }