import { IScopedClusterClient } from 'kibana/server' ; import { DataVisualizer } from '../data_visualizer' ; import { validateJobObject } from './validate_job_object' ; import { CombinedJob } from '../../../common/types/anomaly_detection_jobs' ; import { Detector } from '../../../common/types/anomaly_detection_jobs' ; import { MessageId , JobValidationMessage } from '../../../common/constants/messages' ; import { isValidAggregationField } from '../../../common/util/validation_utils' ; import { getDatafeedAggregations } from '../../../common/util/datafeed_utils' ; function isValidCategorizationConfig ( job , fieldName ) { return ( typeof job . analysis_config . categorization_field_name !== 'undefined' && fieldName === 'mlcategory' ) ; } function isScriptField ( job , fieldName ) { const scriptFields = Object . keys ( job . datafeed_config . script_fields ?? { } ) ; return scriptFields . includes ( fieldName ) ; } function isRuntimeMapping ( job , fieldName ) { const runtimeMappings = Object . keys ( job . datafeed_config . runtime_mappings ?? { } ) ; return runtimeMappings . includes ( fieldName ) ; } const OVER_FIELD_CARDINALITY_THRESHOLD_LOW = 10 ; const OVER_FIELD_CARDINALITY_THRESHOLD_HIGH = 1000000 ; const PARTITION_FIELD_CARDINALITY_THRESHOLD = 1000 ; const BY_FIELD_CARDINALITY_THRESHOLD = 1000 ; const MODEL_PLOT_THRESHOLD_HIGH = 100 ; export type Messages = JobValidationMessage [ ] ; type Validator = ( obj : { type : string ; isInvalid : ( cardinality : number ) => boolean ; messageId ? : MessageId ; } ) => Promise < { modelPlotCardinality : number ; messages : Messages ; } > ; const validateFactory = ( client , job ) : Validator => { const { asCurrentUser } = client ; const dv = new DataVisualizer ( client ) ; const modelPlotConfigTerms = job ?. model_plot_config ?. terms ?? '' ; const modelPlotConfigFieldCount = modelPlotConfigTerms . length > 0 ? modelPlotConfigTerms . split ( ',' ) . length : 0 ; return async ( { type , isInvalid , messageId } ) => { let modelPlotCardinality = 0 ; const messages = [ ] ; const fieldName = `${ type }_field_name` as keyof Pick < Detector , 'by_field_name' | 'over_field_name' | 'partition_field_name' > ; const detectors = job . analysis_config . detectors ; const relevantDetectors = detectors . filter ( ( detector ) => { return typeof detector [ fieldName ] !== 'undefined' ; } ) ; const datafeedConfig = job . datafeed_config ; if ( relevantDetectors . length > 0 ) { try { const uniqueFieldNames = [ ... new Set ( relevantDetectors . map ( ( f ) => f [ fieldName ] ) ) , ] as string [ ] ; const { body : fieldCaps } = await asCurrentUser . fieldCaps ( { index : job . datafeed_config . indices . join ( ',' ) , fields : uniqueFieldNames , } ) ; const datafeedAggregations = getDatafeedAggregations ( datafeedConfig ) ; let aggregatableFieldNames = [ ] ; if ( typeof fieldCaps === 'object' && typeof fieldCaps . fields === 'object' ) { aggregatableFieldNames = uniqueFieldNames . filter ( ( field ) => { if ( typeof datafeedConfig ?. script_fields === 'object' && datafeedConfig ?. script_fields . hasOwnProperty ( field ) ) { return true ; } if ( typeof datafeedConfig ?. runtime_mappings === 'object' && datafeedConfig ?. runtime_mappings . hasOwnProperty ( field ) ) { return true ; } if ( datafeedAggregations !== undefined && isValidAggregationField ( datafeedAggregations , field ) ) { return true ; } if ( typeof fieldCaps . fields [ field ] !== 'undefined' ) { const fieldType = Object . keys ( fieldCaps . fields [ field ] ) [ 0 ] ; return fieldCaps . fields [ field ] [ fieldType ] . aggregatable ; } return false ; } ) ; } const stats = await dv . checkAggregatableFieldsExist ( job . datafeed_config . indices . join ( ',' ) , job . datafeed_config . query , aggregatableFieldNames , 0 , job . data_description . time_field , undefined , undefined , datafeedConfig ) ; if ( stats . totalCount === 0 ) { messages . push ( { id : 'cardinality_no_results' , } ) ; } else { uniqueFieldNames . forEach ( ( uniqueFieldName ) => { const aggregatableNotExistsField = stats . aggregatableNotExistsFields . find ( ( fieldData ) => fieldData . fieldName === uniqueFieldName ) ; if ( aggregatableNotExistsField !== undefined ) { messages . push ( { id : 'cardinality_field_not_exists' , fieldName : uniqueFieldName , } ) ; } else { const field = stats . aggregatableExistsFields . find ( ( fieldData ) => fieldData . fieldName === uniqueFieldName ) ; if ( field !== undefined && typeof field === 'object' && field . stats ) { modelPlotCardinality += modelPlotConfigFieldCount > 0 ? modelPlotConfigFieldCount : field . stats . cardinality ! ; if ( isInvalid ( field . stats . cardinality ! ) ) { messages . push ( { id : messageId || ( `cardinality_${ type }_field` as MessageId ) , fieldName : uniqueFieldName , } ) ; } } else { if ( ! isValidCategorizationConfig ( job , uniqueFieldName ) && ! isScriptField ( job , uniqueFieldName ) && ! isRuntimeMapping ( job , uniqueFieldName ) ) { messages . push ( { id : 'field_not_aggregatable' , fieldName : uniqueFieldName , } ) ; } } } } ) ; } } catch ( e ) { if ( relevantDetectors . length === 1 ) { messages . push ( { id : 'field_not_aggregatable' , fieldName : relevantDetectors [ 0 ] [ fieldName ] ! , } ) ; } else { messages . push ( { id : 'fields_not_aggregatable' } ) ; } return { modelPlotCardinality , messages } ; } } return { modelPlotCardinality , messages } ; } ; } ; export async function validateCardinality ( client , job ? ) { const messages = [ ] ; if ( ! validateJobObject ( job ) ) { throw new Error ( ) ; } const numDetectorsWithFieldNames = job . analysis_config . detectors . filter ( ( d ) => { return d . by_field_name || d . over_field_name || d . partition_field_name ; } ) ; if ( numDetectorsWithFieldNames . length === 0 ) { return [ ] ; } const validate = validateFactory ( client , job ) ; const modelPlotEnabled = job . model_plot_config ?. enabled ?? false ; const validateOverFieldsLow = validate ( { type : 'over' , isInvalid : ( cardinality ) => cardinality < OVER_FIELD_CARDINALITY_THRESHOLD_LOW , messageId : 'cardinality_over_field_low' , } ) ; const validateOverFieldsHigh = validate ( { type : 'over' , isInvalid : ( cardinality ) => cardinality > OVER_FIELD_CARDINALITY_THRESHOLD_HIGH , messageId : 'cardinality_over_field_high' , } ) ; const validatePartitionFields = validate ( { type : 'partition' , isInvalid : ( cardinality ) => cardinality > PARTITION_FIELD_CARDINALITY_THRESHOLD , } ) ; const validateByFields = validate ( { type : 'by' , isInvalid : ( cardinality ) => cardinality > BY_FIELD_CARDINALITY_THRESHOLD , } ) ; const validations = [ await validateByFields , await validateOverFieldsLow , await validateOverFieldsHigh , await validatePartitionFields , ] ; const modelPlotCardinality = validations . reduce ( ( p , c ) => { return p + c . modelPlotCardinality ; } , 0 ) ; if ( modelPlotEnabled && modelPlotCardinality > MODEL_PLOT_THRESHOLD_HIGH ) { messages . push ( { id : 'cardinality_model_plot_high' , modelPlotCardinality , } ) ; } validations . forEach ( ( v ) => messages . push ( ... v . messages ) ) ; if ( messages . length === 0 ) { messages . push ( { id : 'success_cardinality' } ) ; } return messages ; }