import { each , isEmpty , isEqual , pick } from 'lodash' ; import semverGte from 'semver/functions/gte' ; import moment , { Duration } from 'moment' ; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey' ; import numeral from '@elastic/numeral' ; import { i18n } from '@kbn/i18n' ; import { ALLOWED_DATA_UNITS , JOB_ID_MAX_LENGTH } from '../constants/validation' ; import { parseInterval } from './parse_interval' ; import { maxLengthValidator } from './validators' ; import { CREATED_BY_LABEL } from '../constants/new_job' ; import { CombinedJob , CustomSettings , Datafeed , Job , JobId } from '../types/anomaly_detection_jobs' ; import { EntityField } from './anomaly_utils' ; import { MlServerLimits } from '../types/ml_server_info' ; import { JobValidationMessage , JobValidationMessageId } from '../constants/messages' ; import { ES_AGGREGATION , ML_JOB_AGGREGATION } from '../constants/aggregation_types' ; import { MLCATEGORY } from '../constants/field_types' ; import { getAggregations , getDatafeedAggregations } from './datafeed_utils' ; import { findAggField } from './validation_utils' ; import { getFirstKeyInObject , isPopulatedObject } from './object_utils' ; import { isDefined } from '../types/guards' ; export interface ValidationResults { valid : boolean ; messages : JobValidationMessage [ ] ; contains : ( id : JobValidationMessageId ) => boolean ; find : ( id : JobValidationMessageId ) => { id : JobValidationMessageId ; } | undefined ; } export function calculateDatafeedFrequencyDefaultSeconds ( bucketSpanSeconds ) { let freq = 3600 ; if ( bucketSpanSeconds <= 120 ) { freq = 60 ; } else if ( bucketSpanSeconds <= 1200 ) { freq = Math . floor ( bucketSpanSeconds / 2 ) ; } else if ( bucketSpanSeconds <= 43200 ) { freq = 600 ; } return freq ; } export function isTimeSeriesViewJob ( job ) { return getSingleMetricViewerJobErrorMessage ( job ) === undefined ; } export function isTimeSeriesViewDetector ( job , detectorIndex ) { return ( isSourceDataChartableForDetector ( job , detectorIndex ) || isModelPlotChartableForDetector ( job , detectorIndex ) ) ; } export function isMappableJob ( job , detectorIndex ) { let isMappable = false ; const { detectors } = job . analysis_config ; if ( detectorIndex >= 0 && detectorIndex < detectors . length ) { const dtr = detectors [ detectorIndex ] ; const functionName = dtr . function ; isMappable = functionName === ML_JOB_AGGREGATION . LAT_LONG ; } return isMappable ; } param buckets export function hasValidComposite ( buckets ) { if ( isPopulatedObject ( buckets , [ 'composite' ] ) && isPopulatedObject ( buckets . composite , [ 'sources' ] ) && Array . isArray ( buckets . composite . sources ) ) { const sources = buckets . composite . sources ; return ! sources . some ( ( source ) => { const sourceName = getFirstKeyInObject ( source ) ; if ( sourceName !== undefined && isPopulatedObject ( source [ sourceName ] ) ) { const sourceTypes = Object . keys ( source [ sourceName ] ) ; return ( sourceTypes . length === 1 && sourceTypes [ 0 ] !== 'date_histogram' && sourceTypes [ 0 ] !== 'terms' ) ; } return false ; } ) ; } return true ; } param buckets export function isUnsupportedAggType ( aggType ) { return aggType !== 'date_histogram' && aggType !== 'aggs' && aggType !== 'aggregations' ; } export function isSourceDataChartableForDetector ( job , detectorIndex ) { let isSourceDataChartable = false ; const { detectors } = job . analysis_config ; if ( detectorIndex >= 0 && detectorIndex < detectors . length ) { const dtr = detectors [ detectorIndex ] ; const functionName = dtr . function ; isSourceDataChartable = mlFunctionToESAggregation ( functionName ) !== null && dtr . by_field_name !== MLCATEGORY && dtr . partition_field_name !== MLCATEGORY && dtr . over_field_name !== MLCATEGORY ; const hasDatafeed = isPopulatedObject ( job . datafeed_config ) ; if ( isSourceDataChartable && hasDatafeed ) { if ( isPopulatedObject ( job . datafeed_config . script_fields ) ) { const scriptFields = Object . keys ( job . datafeed_config . script_fields ) ; return ( scriptFields . indexOf ( dtr . partition_field_name ! ) === - 1 && scriptFields . indexOf ( dtr . by_field_name ! ) === - 1 && scriptFields . indexOf ( dtr . over_field_name ! ) === - 1 ) ; } const aggs = getDatafeedAggregations ( job . datafeed_config ) ; if ( isPopulatedObject ( aggs ) ) { const aggBucketsName = getFirstKeyInObject ( aggs ) ; if ( aggBucketsName !== undefined ) { if ( Object . keys ( aggs [ aggBucketsName ] ) . some ( isUnsupportedAggType ) ) { return false ; } const aggregations = getAggregations < estypes . AggregationsAggregationContainer > ( aggs [ aggBucketsName ] ) ?? { } ; const foundField = findAggField ( aggregations , dtr . field_name , false ) ; if ( foundField ?. bucket_script !== undefined ) { return false ; } return hasValidComposite ( aggregations ) ; } } return true ; } } return isSourceDataChartable ; } export function isModelPlotChartableForDetector ( job , detectorIndex ) { let isModelPlotChartable = false ; const modelPlotEnabled = job . model_plot_config ?. enabled ?? false ; const { detectors } = job . analysis_config ; if ( detectorIndex >= 0 && detectorIndex < detectors . length && modelPlotEnabled ) { const dtr = detectors [ detectorIndex ] ; const functionName = dtr . function as ML_JOB_AGGREGATION ; isModelPlotChartable = functionName !== ML_JOB_AGGREGATION . RARE && ( mlFunctionToESAggregation ( functionName ) !== null || [ ML_JOB_AGGREGATION . VARP , ML_JOB_AGGREGATION . HIGH_VARP , ML_JOB_AGGREGATION . LOW_VARP , ML_JOB_AGGREGATION . INFO_CONTENT , ML_JOB_AGGREGATION . HIGH_INFO_CONTENT , ML_JOB_AGGREGATION . LOW_INFO_CONTENT , ] . includes ( functionName ) ) ; } return isModelPlotChartable ; } export function getSingleMetricViewerJobErrorMessage ( job ) { const aggs = getDatafeedAggregations ( job . datafeed_config ) ; if ( isPopulatedObject ( aggs ) ) { const aggBucketsName = getFirstKeyInObject ( aggs ) ; if ( aggBucketsName !== undefined && aggs [ aggBucketsName ] !== undefined ) { if ( ! hasValidComposite ( aggs [ aggBucketsName ] ) ) { return i18n . translate ( 'xpack.ml.timeSeriesJob.jobWithUnsupportedCompositeAggregationMessage' , { defaultMessage : 'the datafeed contains unsupported composite sources' , } ) ; } } } const isChartableTimeSeriesViewJob = job . analysis_config . detectors . some ( ( detector , idx ) => isTimeSeriesViewDetector ( job , idx ) ) ; if ( isChartableTimeSeriesViewJob === false ) { return i18n . translate ( 'xpack.ml.timeSeriesJob.notViewableTimeSeriesJobMessage' , { defaultMessage : 'it is not a viewable time series job' , } ) ; } } export function getPartitioningFieldNames ( job , detectorIndex ) { const fieldNames = [ ] ; const detector = job . analysis_config . detectors [ detectorIndex ] ; if ( typeof detector . partition_field_name === 'string' ) { fieldNames . push ( detector . partition_field_name ) ; } if ( typeof detector . by_field_name === 'string' ) { fieldNames . push ( detector . by_field_name ) ; } if ( typeof detector . over_field_name === 'string' ) { fieldNames . push ( detector . over_field_name ) ; } return fieldNames ; } export function isModelPlotEnabled ( job , detectorIndex , entityFields ? ) { let isEnabled = job . model_plot_config ?. enabled ?? false ; if ( isEnabled && entityFields !== undefined && entityFields . length > 0 ) { const termsStr = job . model_plot_config ?. terms ?? '' ; if ( termsStr !== '' ) { const detector = job . analysis_config . detectors [ detectorIndex ] ; const detectorHasPartitionField = detector . hasOwnProperty ( 'partition_field_name' ) ; const detectorHasByField = detector . hasOwnProperty ( 'by_field_name' ) ; const terms = termsStr . split ( ',' ) ; if ( detectorHasPartitionField ) { const partitionEntity = entityFields . find ( ( entityField ) => entityField . fieldName === detector . partition_field_name ) ; isEnabled = partitionEntity ?. fieldValue !== undefined && terms . indexOf ( String ( partitionEntity . fieldValue ) ) !== - 1 ; } if ( isEnabled === true && detectorHasByField === true ) { const byEntity = entityFields . find ( ( entityField ) => entityField . fieldName === detector . by_field_name ) ; isEnabled = byEntity ?. fieldValue !== undefined && terms . indexOf ( String ( byEntity . fieldValue ) ) !== - 1 ; } } } return isEnabled ; } export function isJobVersionGte ( job , version ) { const jobVersion = job . job_version ?? '0.0.0' ; return semverGte ( jobVersion , version ) ; } export function mlFunctionToESAggregation ( functionName ) { if ( functionName === ML_JOB_AGGREGATION . MEAN || functionName === ML_JOB_AGGREGATION . HIGH_MEAN || functionName === ML_JOB_AGGREGATION . LOW_MEAN || functionName === ML_JOB_AGGREGATION . METRIC ) { return ES_AGGREGATION . AVG ; } if ( functionName === ML_JOB_AGGREGATION . SUM || functionName === ML_JOB_AGGREGATION . HIGH_SUM || functionName === ML_JOB_AGGREGATION . LOW_SUM || functionName === ML_JOB_AGGREGATION . NON_NULL_SUM || functionName === ML_JOB_AGGREGATION . LOW_NON_NULL_SUM || functionName === ML_JOB_AGGREGATION . HIGH_NON_NULL_SUM ) { return ES_AGGREGATION . SUM ; } if ( functionName === ML_JOB_AGGREGATION . COUNT || functionName === ML_JOB_AGGREGATION . HIGH_COUNT || functionName === ML_JOB_AGGREGATION . LOW_COUNT || functionName === ML_JOB_AGGREGATION . NON_ZERO_COUNT || functionName === ML_JOB_AGGREGATION . LOW_NON_ZERO_COUNT || functionName === ML_JOB_AGGREGATION . HIGH_NON_ZERO_COUNT ) { return ES_AGGREGATION . COUNT ; } if ( functionName === ML_JOB_AGGREGATION . DISTINCT_COUNT || functionName === ML_JOB_AGGREGATION . LOW_DISTINCT_COUNT || functionName === ML_JOB_AGGREGATION . HIGH_DISTINCT_COUNT ) { return ES_AGGREGATION . CARDINALITY ; } if ( functionName === ML_JOB_AGGREGATION . MEDIAN || functionName === ML_JOB_AGGREGATION . HIGH_MEDIAN || functionName === ML_JOB_AGGREGATION . LOW_MEDIAN ) { return ES_AGGREGATION . PERCENTILES ; } if ( functionName === ML_JOB_AGGREGATION . MIN || functionName === ML_JOB_AGGREGATION . MAX ) { return functionName as unknown as ES_AGGREGATION ; } if ( functionName === ML_JOB_AGGREGATION . RARE ) { return ES_AGGREGATION . COUNT ; } return null ; } export function isJobIdValid ( jobId ) { return /^[a-z0-9\-\_]+$/g . test ( jobId ) && ! /^([_-].*)?(.*[_-])?$/g . test ( jobId ) ; } export const ML_MEDIAN_PERCENTS = '50.0' ; export const ML_DATA_PREVIEW_COUNT = 10 ; export function prefixDatafeedId ( datafeedId , prefix ) { return datafeedId . match ( /^datafeed-/ ) ? datafeedId . replace ( /^datafeed-/ , `datafeed-${ prefix }` ) : `datafeed-${ prefix }${ datafeedId }` ; } export function getSafeAggregationName ( fieldName , index ) { return fieldName . match ( /^[a-zA-Z0-9-_.]+$/ ) ? fieldName : `field_${ index }` ; } export function uniqWithIsEqual < T extends any [ ] > ( arr ) { return arr . reduce ( ( dedupedArray , value ) => { if ( dedupedArray . filter ( ( compareValue ) => isEqual ( compareValue , value ) ) . length === 0 ) { dedupedArray . push ( value ) ; } return dedupedArray ; } , [ ] ) ; } export function basicJobValidation ( job , fields , limits , skipMmlChecks = false ) { const messages = [ ] ; let valid = true ; if ( job ) { if ( isEmpty ( job . job_id ) ) { messages . push ( { id : 'job_id_empty' } ) ; valid = false ; } else if ( isJobIdValid ( job . job_id ) === false ) { messages . push ( { id : 'job_id_invalid' } ) ; valid = false ; } else if ( maxLengthValidator ( JOB_ID_MAX_LENGTH ) ( job . job_id ) ) { messages . push ( { id : 'job_id_invalid_max_length' , maxLength : JOB_ID_MAX_LENGTH } ) ; valid = false ; } else { messages . push ( { id : 'job_id_valid' } ) ; } const { messages : groupsMessages , valid : groupsValid } = validateGroupNames ( job ) ; messages . push ( ... groupsMessages ) ; valid = valid && groupsValid ; if ( job . analysis_config . categorization_filters ) { let v = true ; each ( job . analysis_config . categorization_filters , ( d ) => { try { new RegExp ( d ) ; } catch ( e ) { v = false ; } if ( job . analysis_config . categorization_field_name === undefined || job . analysis_config . categorization_field_name === '' ) { v = false ; } if ( d === '' ) { v = false ; } } ) ; if ( v ) { messages . push ( { id : 'categorization_filters_valid' } ) ; } else { messages . push ( { id : 'categorization_filters_invalid' } ) ; valid = false ; } } let categorizerDetectorMissingPartitionField = false ; if ( job . analysis_config . detectors . length === 0 ) { messages . push ( { id : 'detectors_empty' } ) ; valid = false ; } else { let v = true ; each ( job . analysis_config . detectors , ( d ) => { if ( isEmpty ( d . function ) ) { v = false ; } const needToHavePartitionFieldName = job . analysis_config . per_partition_categorization ?. enabled === true && ( d . by_field_name === MLCATEGORY || d . over_field_name === MLCATEGORY ) ; if ( needToHavePartitionFieldName && d . partition_field_name === undefined ) { categorizerDetectorMissingPartitionField = true ; } } ) ; if ( v ) { messages . push ( { id : 'detectors_function_not_empty' } ) ; } else { messages . push ( { id : 'detectors_function_empty' } ) ; valid = false ; } if ( categorizerDetectorMissingPartitionField ) { messages . push ( { id : 'categorizer_detector_missing_per_partition_field' } ) ; valid = false ; } } if ( job . analysis_config . detectors . length >= 2 ) { if ( job . analysis_config . per_partition_categorization !== undefined ) { if ( job . analysis_config . per_partition_categorization . enabled || ( job . analysis_config . per_partition_categorization . stop_on_warn && Array . isArray ( job . analysis_config . detectors ) && job . analysis_config . detectors . length >= 2 ) ) { const categorizationDetectors = job . analysis_config . detectors . filter ( ( d ) => d . by_field_name === MLCATEGORY || d . over_field_name === MLCATEGORY || d . partition_field_name === MLCATEGORY ) ; const uniqPartitions = [ ... new Set ( categorizationDetectors . map ( ( d ) => d . partition_field_name ) . filter ( ( name ) => name !== undefined ) ) , ] ; if ( uniqPartitions . length > 1 ) { valid = false ; messages . push ( { id : 'categorizer_varying_per_partition_fields' , fields : uniqPartitions . join ( ', ' ) , } ) ; } } } const compareSubSet = job . analysis_config . detectors . map ( ( d ) => pick ( d , [ 'function' , 'field_name' , 'by_field_name' , 'over_field_name' , 'partition_field_name' , ] ) ) ; const dedupedSubSet = uniqWithIsEqual ( compareSubSet ) ; if ( compareSubSet . length !== dedupedSubSet . length ) { messages . push ( { id : 'detectors_duplicates' } ) ; valid = false ; } } if ( job . analysis_config . bucket_span === '' || job . analysis_config . bucket_span === undefined ) { messages . push ( { id : 'bucket_span_empty' } ) ; valid = false ; } else { if ( isValidTimeInterval ( job . analysis_config . bucket_span ) ) { messages . push ( { id : 'bucket_span_valid' , bucketSpan : job . analysis_config . bucket_span , } ) ; } else { messages . push ( { id : 'bucket_span_invalid' } ) ; valid = false ; } } if ( typeof fields !== 'undefined' ) { const loadedFields = Object . keys ( fields ) ; if ( loadedFields . length === 0 ) { messages . push ( { id : 'index_fields_invalid' } ) ; valid = false ; } else { messages . push ( { id : 'index_fields_valid' } ) ; } } if ( skipMmlChecks === false ) { const mml = job . analysis_limits && job . analysis_limits . model_memory_limit ; const { messages : mmlUnitMessages , valid : mmlUnitValid } = validateModelMemoryLimitUnits ( mml ) ; messages . push ( ... mmlUnitMessages ) ; valid = valid && mmlUnitValid ; if ( mmlUnitValid ) { const { messages : mmlMessages , valid : mmlValid } = validateModelMemoryLimit ( job , limits ) ; messages . push ( ... mmlMessages ) ; valid = valid && mmlValid ; } } } else { valid = false ; } return { messages , valid , contains : ( id ) => messages . some ( ( m ) => id === m . id ) , find : ( id ) => messages . find ( ( m ) => id === m . id ) , } ; } export function basicDatafeedValidation ( datafeed ) { const messages = [ ] ; let valid = true ; if ( datafeed ) { let queryDelayMessage = { id : 'query_delay_valid' } ; if ( isValidTimeInterval ( datafeed . query_delay ) === false ) { queryDelayMessage = { id : 'query_delay_invalid' } ; valid = false ; } messages . push ( queryDelayMessage ) ; let frequencyMessage = { id : 'frequency_valid' } ; if ( isValidTimeInterval ( datafeed . frequency ) === false ) { frequencyMessage = { id : 'frequency_invalid' } ; valid = false ; } messages . push ( frequencyMessage ) ; } return { messages , valid , contains : ( id ) => messages . some ( ( m ) => id === m . id ) , find : ( id ) => messages . find ( ( m ) => id === m . id ) , } ; } export function basicJobAndDatafeedValidation ( job , datafeed ) { const messages = [ ] ; let valid = true ; if ( datafeed && job ) { const datafeedAggregations = getDatafeedAggregations ( datafeed ) ; if ( datafeedAggregations !== undefined && ! job . analysis_config ?. summary_count_field_name ) { valid = false ; messages . push ( { id : 'missing_summary_count_field_name' } ) ; } } return { messages , valid , contains : ( id ) => messages . some ( ( m ) => id === m . id ) , find : ( id ) => messages . find ( ( m ) => id === m . id ) , } ; } export function validateModelMemoryLimit ( job , limits ) { const messages = [ ] ; let valid = true ; if ( typeof job . analysis_limits !== 'undefined' && typeof job . analysis_limits . model_memory_limit !== 'undefined' ) { if ( typeof limits === 'object' && typeof limits . max_model_memory_limit !== 'undefined' ) { const max = limits . max_model_memory_limit . toUpperCase ( ) ; const mml = job . analysis_limits . model_memory_limit . toUpperCase ( ) ; const mmlBytes = numeral ( mml ) . value ( ) ; const maxBytes = numeral ( max ) . value ( ) ; if ( mmlBytes > maxBytes ) { messages . push ( { id : 'model_memory_limit_invalid' } ) ; valid = false ; } else { messages . push ( { id : 'model_memory_limit_valid' } ) ; } } } return { valid , messages , contains : ( id ) => messages . some ( ( m ) => id === m . id ) , find : ( id ) => messages . find ( ( m ) => id === m . id ) , } ; } export function validateModelMemoryLimitUnits ( modelMemoryLimit ) { const messages = [ ] ; let valid = true ; if ( modelMemoryLimit !== undefined ) { const mml = String ( modelMemoryLimit ) . toUpperCase ( ) ; const mmlSplit = mml . match ( /\d+(\w+)$/ ) ; const unit = mmlSplit && mmlSplit . length === 2 ? mmlSplit [ 1 ] : null ; if ( unit === null || ALLOWED_DATA_UNITS . indexOf ( unit ) === - 1 ) { messages . push ( { id : 'model_memory_limit_units_invalid' } ) ; valid = false ; } else { messages . push ( { id : 'model_memory_limit_units_valid' } ) ; } } return { valid , messages , contains : ( id ) => messages . some ( ( m ) => id === m . id ) , find : ( id ) => messages . find ( ( m ) => id === m . id ) , } ; } export function validateGroupNames ( job ) { const { groups = [ ] } = job ; const errorMessages = [ ... ( groups . some ( ( group ) => ! isJobIdValid ( group ) ) ? [ { id : 'job_group_id_invalid' } ] : [ ] ) , ... ( groups . some ( ( group ) => maxLengthValidator ( JOB_ID_MAX_LENGTH ) ( group ) ) ? [ { id : 'job_group_id_invalid_max_length' } ] : [ ] ) , ] ; const valid = errorMessages . length === 0 ; const messages = valid && groups . length ? [ { id : 'job_group_id_valid' } ] : errorMessages ; return { valid , messages , contains : ( id ) => messages . some ( ( m ) => id === m . id ) , find : ( id ) => messages . find ( ( m ) => id === m . id ) , } ; } param value return { Duration } export function parseTimeIntervalForJob ( value ) { if ( value === undefined ) { return null ; } const interval = parseInterval ( value , true ) ; return interval !== null && interval . asMilliseconds ( ) !== 0 && interval . milliseconds ( ) === 0 ? interval : null ; } function isValidTimeInterval ( value ) { if ( value === undefined ) { return true ; } return parseTimeIntervalForJob ( value ) !== null ; } export function getEarliestDatafeedStartTime ( latestRecordTimestamp , latestBucketTimestamp , bucketSpan ? ) { if ( latestRecordTimestamp !== undefined && latestBucketTimestamp !== undefined ) { const adjustedBucketStartTime = bucketSpan ? moment ( latestBucketTimestamp ) . add ( bucketSpan ) . valueOf ( ) : latestBucketTimestamp ; return Math . max ( latestRecordTimestamp , adjustedBucketStartTime ) ; } else { return latestRecordTimestamp !== undefined ? latestRecordTimestamp : latestBucketTimestamp ; } } export function getLatestDataOrBucketTimestamp ( latestDataTimestamp , latestBucketTimestamp ) { if ( latestDataTimestamp !== undefined && latestBucketTimestamp !== undefined ) { return Math . max ( latestDataTimestamp , latestBucketTimestamp ) ; } else { return latestDataTimestamp !== undefined ? latestDataTimestamp : latestBucketTimestamp ; } } /**
 * If created_by is set in the job's custom_settings, remove it in case
 * it was created by a job wizard as the rules cannot currently be edited
 * in the job wizards and so would be lost in a clone.
 */ export function processCreatedBy ( customSettings ) { if ( Object . values ( CREATED_BY_LABEL ) . includes ( customSettings . created_by as CREATED_BY_LABEL ) ) { delete customSettings . created_by ; } } export function splitIndexPatternNames ( indexPatternName ) { return indexPatternName . includes ( ',' ) ? indexPatternName . split ( ',' ) . map ( ( i ) => i . trim ( ) ) : [ indexPatternName ] ; } param timeIntervals export function resolveMaxTimeInterval ( timeIntervals ) { const result = Math . max ( ... timeIntervals . map ( ( b ) => parseInterval ( b ) ) . filter ( isDefined ) . map ( ( v ) => v . asSeconds ( ) ) ) ; return Number . isFinite ( result ) ? result : undefined ; }