import { IScopedClusterClient } from 'kibana/server' ; import { get , each , last , find } from 'lodash' ; import { KBN_FIELD_TYPES } from '../../../../../../src/plugins/data/server' ; import { ML_JOB_FIELD_TYPES } from '../../../common/constants/field_types' ; import { getSafeAggregationName } from '../../../common/util/job_utils' ; import { stringHash } from '../../../common/util/string_utils' ; import { buildBaseFilterCriteria , buildSamplerAggregation , getSamplerAggregationsResponsePath , } from '../../lib/query_utils' ; import { AggCardinality , RuntimeMappings } from '../../../common/types/fields' ; import { getDatafeedAggregations } from '../../../common/util/datafeed_utils' ; import { Datafeed } from '../../../common/types/anomaly_detection_jobs' ; import { isPopulatedObject } from '../../../common/util/object_utils' ; const SAMPLER_TOP_TERMS_THRESHOLD = 100000 ; const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000 ; const AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE = 200 ; const FIELDS_REQUEST_BATCH_SIZE = 10 ; const MAX_CHART_COLUMNS = 20 ; interface FieldData { fieldName : string ; existsInDocs : boolean ; stats ? : { sampleCount ? : number ; count ? : number ; cardinality ? : number ; } ; } export interface Field { fieldName : string ; type : string ; cardinality : number ; } export interface HistogramField { fieldName : string ; type : string ; } interface Distribution { percentiles : any [ ] ; minPercentile : number ; maxPercentile : number ; } interface Aggs { [ key ] : any ; } interface Bucket { doc_count : number ; } interface NumericFieldStats { fieldName : string ; count : number ; min : number ; max : number ; avg : number ; isTopValuesSampled : boolean ; topValues : Bucket [ ] ; topValuesSampleSize : number ; topValuesSamplerShardSize : number ; median ? : number ; distribution ? : Distribution ; } interface StringFieldStats { fieldName : string ; isTopValuesSampled : boolean ; topValues : Bucket [ ] ; topValuesSampleSize : number ; topValuesSamplerShardSize : number ; } interface DateFieldStats { fieldName : string ; count : number ; earliest : number ; latest : number ; } interface BooleanFieldStats { fieldName : string ; count : number ; trueCount : number ; falseCount : number ; [ key ] : number | string ; } interface DocumentCountStats { documentCounts : { interval : number ; buckets : { [ key ] : number ; } ; } ; } interface FieldExamples { fieldName : string ; examples : any [ ] ; } interface NumericColumnStats { interval : number ; min : number ; max : number ; } type NumericColumnStatsMap = Record < string , NumericColumnStats > ; interface AggHistogram { histogram : { field : string ; interval : number ; } ; } interface AggTerms { terms : { field : string ; size : number ; } ; } interface NumericDataItem { key : number ; key_as_string ? : string ; doc_count : number ; } interface NumericChartData { data : NumericDataItem [ ] ; id : string ; interval : number ; stats : [ number , number ] ; type : 'numeric' ; } interface OrdinalDataItem { key : string ; key_as_string ? : string ; doc_count : number ; } interface OrdinalChartData { type : 'ordinal' | 'boolean' ; cardinality : number ; data : OrdinalDataItem [ ] ; id : string ; } interface UnsupportedChartData { id : string ; type : 'unsupported' ; } type ChartRequestAgg = AggHistogram | AggCardinality | AggTerms ; type ChartData = NumericChartData | OrdinalChartData | UnsupportedChartData ; type BatchStats = NumericFieldStats | StringFieldStats | BooleanFieldStats | DateFieldStats | DocumentCountStats | FieldExamples ; const getAggIntervals = async ( { asCurrentUser } , indexPattern , query , fields , samplerShardSize , runtimeMappings ? ) : Promise < NumericColumnStatsMap > => { const numericColumns = fields . filter ( ( field ) => { return field . type === KBN_FIELD_TYPES . NUMBER || field . type === KBN_FIELD_TYPES . DATE ; } ) ; if ( numericColumns . length === 0 ) { return { } ; } const minMaxAggs = numericColumns . reduce ( ( aggs , c ) => { const id = stringHash ( c . fieldName ) ; aggs [ id ] = { stats : { field : c . fieldName , } , } ; return aggs ; } , { } as Record < string , object > ) ; const { body } = await asCurrentUser . search ( { index : indexPattern , size : 0 , body : { query , aggs : buildSamplerAggregation ( minMaxAggs , samplerShardSize ) , size : 0 , ... ( isPopulatedObject ( runtimeMappings ) ? { runtime_mappings : runtimeMappings } : { } ) , } , } ) ; const aggsPath = getSamplerAggregationsResponsePath ( samplerShardSize ) ; const aggregations = aggsPath . length > 0 ? get ( body . aggregations , aggsPath ) : body . aggregations ; return Object . keys ( aggregations ) . reduce ( ( p , aggName ) => { const stats = [ aggregations [ aggName ] . min , aggregations [ aggName ] . max ] ; if ( ! stats . includes ( null ) ) { const delta = aggregations [ aggName ] . max - aggregations [ aggName ] . min ; let aggInterval = 1 ; if ( delta > MAX_CHART_COLUMNS || delta <= 1 ) { aggInterval = delta / ( MAX_CHART_COLUMNS - 1 ) ; } p [ aggName ] = { interval : aggInterval , min : stats [ 0 ] , max : stats [ 1 ] } ; } return p ; } , { } as NumericColumnStatsMap ) ; } ; export const getHistogramsForFields = async ( client , indexPattern , query , fields , samplerShardSize , runtimeMappings ? ) => { const { asCurrentUser } = client ; const aggIntervals = await getAggIntervals ( client , indexPattern , query , fields , samplerShardSize , runtimeMappings ) ; const chartDataAggs = fields . reduce ( ( aggs , field ) => { const fieldName = field . fieldName ; const fieldType = field . type ; const id = stringHash ( fieldName ) ; if ( fieldType === KBN_FIELD_TYPES . NUMBER || fieldType === KBN_FIELD_TYPES . DATE ) { if ( aggIntervals [ id ] !== undefined ) { aggs [ `${ id }_histogram` ] = { histogram : { field : fieldName , interval : aggIntervals [ id ] . interval !== 0 ? aggIntervals [ id ] . interval : 1 , } , } ; } } else if ( fieldType === KBN_FIELD_TYPES . STRING || fieldType === KBN_FIELD_TYPES . BOOLEAN ) { if ( fieldType === KBN_FIELD_TYPES . STRING ) { aggs [ `${ id }_cardinality` ] = { cardinality : { field : fieldName , } , } ; } aggs [ `${ id }_terms` ] = { terms : { field : fieldName , size : MAX_CHART_COLUMNS , } , } ; } return aggs ; } , { } as Record < string , ChartRequestAgg > ) ; if ( Object . keys ( chartDataAggs ) . length === 0 ) { return [ ] ; } const { body } = await asCurrentUser . search ( { index : indexPattern , size : 0 , body : { query , aggs : buildSamplerAggregation ( chartDataAggs , samplerShardSize ) , size : 0 , ... ( isPopulatedObject ( runtimeMappings ) ? { runtime_mappings : runtimeMappings } : { } ) , } , } ) ; const aggsPath = getSamplerAggregationsResponsePath ( samplerShardSize ) ; const aggregations = aggsPath . length > 0 ? get ( body . aggregations , aggsPath ) : body . aggregations ; const chartsData = fields . map ( ( field ) : ChartData => { const fieldName = field . fieldName ; const fieldType = field . type ; const id = stringHash ( field . fieldName ) ; if ( fieldType === KBN_FIELD_TYPES . NUMBER || fieldType === KBN_FIELD_TYPES . DATE ) { if ( aggIntervals [ id ] === undefined ) { return { type : 'numeric' , data : [ ] , interval : 0 , stats : [ 0 , 0 ] , id : fieldName , } ; } return { data : aggregations [ `${ id }_histogram` ] . buckets , interval : aggIntervals [ id ] . interval , stats : [ aggIntervals [ id ] . min , aggIntervals [ id ] . max ] , type : 'numeric' , id : fieldName , } ; } else if ( fieldType === KBN_FIELD_TYPES . STRING || fieldType === KBN_FIELD_TYPES . BOOLEAN ) { return { type : fieldType === KBN_FIELD_TYPES . STRING ? 'ordinal' : 'boolean' , cardinality : fieldType === KBN_FIELD_TYPES . STRING ? aggregations [ `${ id }_cardinality` ] . value : 2 , data : aggregations [ `${ id }_terms` ] . buckets , id : fieldName , } ; } return { type : 'unsupported' , id : fieldName , } ; } ) ; return chartsData ; } ; export class DataVisualizer { private _client : IScopedClusterClient ; private _asCurrentUser : IScopedClusterClient [ 'asCurrentUser' ] ; constructor ( client ) { this . _asCurrentUser = client . asCurrentUser ; this . _client = client ; } async getOverallStats ( indexPatternTitle , query , aggregatableFields , nonAggregatableFields , samplerShardSize , timeFieldName , earliestMs , latestMs , runtimeMappings ? ) { const stats = { totalCount : 0 , aggregatableExistsFields : [ ] as FieldData [ ] , aggregatableNotExistsFields : [ ] as FieldData [ ] , nonAggregatableExistsFields : [ ] as FieldData [ ] , nonAggregatableNotExistsFields : [ ] as FieldData [ ] , } ; const batches = [ [ ] ] ; each ( aggregatableFields , ( field ) => { let lastArray = last ( batches ) as string [ ] ; if ( lastArray . length === AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE ) { lastArray = [ ] ; batches . push ( lastArray ) ; } lastArray . push ( field ) ; } ) ; await Promise . all ( batches . map ( async ( fields ) => { const batchStats = await this . checkAggregatableFieldsExist ( indexPatternTitle , query , fields , samplerShardSize , timeFieldName , earliestMs , latestMs , undefined , runtimeMappings ) ; stats . totalCount = batchStats . totalCount ; stats . aggregatableExistsFields . push ( ... batchStats . aggregatableExistsFields ) ; stats . aggregatableNotExistsFields . push ( ... batchStats . aggregatableNotExistsFields ) ; } ) ) ; await Promise . all ( nonAggregatableFields . map ( async ( field ) => { const existsInDocs = await this . checkNonAggregatableFieldExists ( indexPatternTitle , query , field , timeFieldName , earliestMs , latestMs , runtimeMappings ) ; const fieldData = { fieldName : field , existsInDocs , stats : { } , } ; if ( existsInDocs === true ) { stats . nonAggregatableExistsFields . push ( fieldData ) ; } else { stats . nonAggregatableNotExistsFields . push ( fieldData ) ; } } ) ) ; return stats ; } async getHistogramsForFields ( indexPattern , query , fields , samplerShardSize , runtimeMappings ? ) { return await getHistogramsForFields ( this . _client , indexPattern , query , fields , samplerShardSize , runtimeMappings ) ; } async getStatsForFields ( indexPatternTitle , query , fields , samplerShardSize , timeFieldName , earliestMs , latestMs , intervalMs , maxExamples , runtimeMappings ) { const batches = [ ] ; const batchedFields = { } ; each ( fields , ( field ) => { if ( field . fieldName === undefined ) { if ( timeFieldName !== undefined ) { batches . push ( [ field ] ) ; } } else { const fieldType = field . type ; if ( batchedFields [ fieldType ] === undefined ) { batchedFields [ fieldType ] = [ [ ] ] ; } let lastArray = last ( batchedFields [ fieldType ] ) as Field [ ] ; if ( lastArray . length === FIELDS_REQUEST_BATCH_SIZE ) { lastArray = [ ] ; batchedFields [ fieldType ] . push ( lastArray ) ; } lastArray . push ( field ) ; } } ) ; each ( batchedFields , ( lists ) => { batches . push ( ... lists ) ; } ) ; let results = [ ] ; await Promise . all ( batches . map ( async ( batch ) => { let batchStats = [ ] ; const first = batch [ 0 ] ; switch ( first . type ) { case ML_JOB_FIELD_TYPES . NUMBER : if ( first . fieldName !== undefined ) { batchStats = await this . getNumericFieldsStats ( indexPatternTitle , query , batch , samplerShardSize , timeFieldName , earliestMs , latestMs , runtimeMappings ) ; } else { if ( intervalMs !== undefined ) { const stats = await this . getDocumentCountStats ( indexPatternTitle , query , timeFieldName , earliestMs , latestMs , intervalMs , runtimeMappings ) ; batchStats . push ( stats ) ; } } break ; case ML_JOB_FIELD_TYPES . KEYWORD : case ML_JOB_FIELD_TYPES . IP : batchStats = await this . getStringFieldsStats ( indexPatternTitle , query , batch , samplerShardSize , timeFieldName , earliestMs , latestMs , runtimeMappings ) ; break ; case ML_JOB_FIELD_TYPES . DATE : batchStats = await this . getDateFieldsStats ( indexPatternTitle , query , batch , samplerShardSize , timeFieldName , earliestMs , latestMs , runtimeMappings ) ; break ; case ML_JOB_FIELD_TYPES . BOOLEAN : batchStats = await this . getBooleanFieldsStats ( indexPatternTitle , query , batch , samplerShardSize , timeFieldName , earliestMs , latestMs , runtimeMappings ) ; break ; case ML_JOB_FIELD_TYPES . TEXT : default : await Promise . all ( batch . map ( async ( field ) => { const stats = await this . getFieldExamples ( indexPatternTitle , query , field . fieldName , timeFieldName , earliestMs , latestMs , maxExamples , runtimeMappings ) ; batchStats . push ( stats ) ; } ) ) ; break ; } results = [ ... results , ... batchStats ] ; } ) ) ; return results ; } async checkAggregatableFieldsExist ( indexPatternTitle , query , aggregatableFields , samplerShardSize , timeFieldName , earliestMs ? , latestMs ? , datafeedConfig ? , runtimeMappings ? ) { const index = indexPatternTitle ; const size = 0 ; const filterCriteria = buildBaseFilterCriteria ( timeFieldName , earliestMs , latestMs , query ) ; const datafeedAggregations = getDatafeedAggregations ( datafeedConfig ) ; const aggs = datafeedAggregations !== undefined ? { ... datafeedAggregations } : { } ; const combinedRuntimeMappings = { ... ( isPopulatedObject ( runtimeMappings ) ? runtimeMappings : { } ) , ... ( isPopulatedObject ( datafeedConfig ) && isPopulatedObject ( datafeedConfig . runtime_mappings ) ? datafeedConfig . runtime_mappings : { } ) , } ; aggregatableFields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field , i ) ; aggs [ `${ safeFieldName }_count` ] = { filter : { exists : { field } } , } ; let cardinalityField ; if ( datafeedConfig ?. script_fields ?. hasOwnProperty ( field ) ) { cardinalityField = aggs [ `${ safeFieldName }_cardinality` ] = { cardinality : { script : datafeedConfig ?. script_fields [ field ] . script } , } ; } else { cardinalityField = { cardinality : { field } , } ; } aggs [ `${ safeFieldName }_cardinality` ] = cardinalityField ; } ) ; const searchBody = { query : { bool : { filter : filterCriteria , } , } , ... ( isPopulatedObject ( aggs ) ? { aggs : buildSamplerAggregation ( aggs , samplerShardSize ) } : { } ) , ... ( isPopulatedObject ( combinedRuntimeMappings ) ? { runtime_mappings : combinedRuntimeMappings } : { } ) , } ; const { body } = await this . _asCurrentUser . search ( { index , track_total_hits : true , size , body : searchBody , } ) ; const aggregations = body . aggregations ; const totalCount = body . hits . total . value ; const stats = { totalCount , aggregatableExistsFields : [ ] as FieldData [ ] , aggregatableNotExistsFields : [ ] as FieldData [ ] , } ; const aggsPath = getSamplerAggregationsResponsePath ( samplerShardSize ) ; const sampleCount = samplerShardSize > 0 ? get ( aggregations , [ 'sample' , 'doc_count' ] , 0 ) : totalCount ; aggregatableFields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field , i ) ; const count = get ( aggregations , [ ... aggsPath , `${ safeFieldName }_count` , 'doc_count' ] , 0 ) ; if ( count > 0 ) { const cardinality = get ( aggregations , [ ... aggsPath , `${ safeFieldName }_cardinality` , 'value' ] , 0 ) ; stats . aggregatableExistsFields . push ( { fieldName : field , existsInDocs : true , stats : { sampleCount , count , cardinality , } , } ) ; } else { if ( datafeedConfig ?. script_fields ?. hasOwnProperty ( field ) || datafeedConfig ?. runtime_mappings ?. hasOwnProperty ( field ) ) { const cardinality = get ( aggregations , [ ... aggsPath , `${ safeFieldName }_cardinality` , 'value' ] , 0 ) ; stats . aggregatableExistsFields . push ( { fieldName : field , existsInDocs : true , stats : { sampleCount , count , cardinality , } , } ) ; } else { stats . aggregatableNotExistsFields . push ( { fieldName : field , existsInDocs : false , } ) ; } } } ) ; return stats ; } async checkNonAggregatableFieldExists ( indexPatternTitle , query , field , timeFieldName , earliestMs , latestMs , runtimeMappings ? ) { const index = indexPatternTitle ; const size = 0 ; const filterCriteria = buildBaseFilterCriteria ( timeFieldName , earliestMs , latestMs , query ) ; const searchBody = { query : { bool : { filter : filterCriteria , } , } , ... ( isPopulatedObject ( runtimeMappings ) ? { runtime_mappings : runtimeMappings } : { } ) , } ; filterCriteria . push ( { exists : { field } } ) ; const { body } = await this . _asCurrentUser . search ( { index , size , body : searchBody , } ) ; return body . hits . total . value > 0 ; } async getDocumentCountStats ( indexPatternTitle , query , timeFieldName , earliestMs , latestMs , intervalMs , runtimeMappings ) { const index = indexPatternTitle ; const size = 0 ; const filterCriteria = buildBaseFilterCriteria ( timeFieldName , earliestMs , latestMs , query ) ; const aggs = { eventRate : { date_histogram : { field : timeFieldName , fixed_interval : `${ intervalMs }ms` , min_doc_count : 1 , } , } , } ; const searchBody = { query : { bool : { filter : filterCriteria , } , } , aggs , ... ( isPopulatedObject ( runtimeMappings ) ? { runtime_mappings : runtimeMappings } : { } ) , } ; const { body } = await this . _asCurrentUser . search ( { index , size , body : searchBody , } ) ; const buckets = { } ; const dataByTimeBucket = get ( body , [ 'aggregations' , 'eventRate' , 'buckets' ] , [ ] ) ; each ( dataByTimeBucket , ( dataForTime ) => { const time = dataForTime . key ; buckets [ time ] = dataForTime . doc_count ; } ) ; return { documentCounts : { interval : intervalMs , buckets , } , } ; } async getNumericFieldsStats ( indexPatternTitle , query , fields , samplerShardSize , timeFieldName , earliestMs , latestMs , runtimeMappings ? ) { const index = indexPatternTitle ; const size = 0 ; const filterCriteria = buildBaseFilterCriteria ( timeFieldName , earliestMs , latestMs , query ) ; const MAX_PERCENT = 100 ; const PERCENTILE_SPACING = 5 ; let count = 0 ; const percents = Array . from ( Array ( MAX_PERCENT / PERCENTILE_SPACING ) , ( ) => ( count += PERCENTILE_SPACING ) ) ; const aggs = { } ; fields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field . fieldName , i ) ; aggs [ `${ safeFieldName }_field_stats` ] = { filter : { exists : { field : field . fieldName } } , aggs : { actual_stats : { stats : { field : field . fieldName } , } , } , } ; aggs [ `${ safeFieldName }_percentiles` ] = { percentiles : { field : field . fieldName , percents , keyed : false , } , } ; const top = { terms : { field : field . fieldName , size : 10 , order : { _count : 'desc' , } , } , } ; if ( samplerShardSize < 1 && field . cardinality >= SAMPLER_TOP_TERMS_THRESHOLD ) { aggs [ `${ safeFieldName }_top` ] = { sampler : { shard_size : SAMPLER_TOP_TERMS_SHARD_SIZE , } , aggs : { top , } , } ; } else { aggs [ `${ safeFieldName }_top` ] = top ; } } ) ; const searchBody = { query : { bool : { filter : filterCriteria , } , } , aggs : buildSamplerAggregation ( aggs , samplerShardSize ) , ... ( isPopulatedObject ( runtimeMappings ) ? { runtime_mappings : runtimeMappings } : { } ) , } ; const { body } = await this . _asCurrentUser . search ( { index , size , body : searchBody , } ) ; const aggregations = body . aggregations ; const aggsPath = getSamplerAggregationsResponsePath ( samplerShardSize ) ; const batchStats = [ ] ; fields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field . fieldName , i ) ; const docCount = get ( aggregations , [ ... aggsPath , `${ safeFieldName }_field_stats` , 'doc_count' ] , 0 ) ; const fieldStatsResp = get ( aggregations , [ ... aggsPath , `${ safeFieldName }_field_stats` , 'actual_stats' ] , { } ) ; const topAggsPath = [ ... aggsPath , `${ safeFieldName }_top` ] ; if ( samplerShardSize < 1 && field . cardinality >= SAMPLER_TOP_TERMS_THRESHOLD ) { topAggsPath . push ( 'top' ) ; } const topValues = get ( aggregations , [ ... topAggsPath , 'buckets' ] , [ ] ) ; const stats = { fieldName : field . fieldName , count : docCount , min : get ( fieldStatsResp , 'min' , 0 ) , max : get ( fieldStatsResp , 'max' , 0 ) , avg : get ( fieldStatsResp , 'avg' , 0 ) , isTopValuesSampled : field . cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0 , topValues , topValuesSampleSize : topValues . reduce ( ( acc , curr ) => acc + curr . doc_count , get ( aggregations , [ ... topAggsPath , 'sum_other_doc_count' ] , 0 ) ) , topValuesSamplerShardSize : field . cardinality >= SAMPLER_TOP_TERMS_THRESHOLD ? SAMPLER_TOP_TERMS_SHARD_SIZE : samplerShardSize , } ; if ( stats . count > 0 ) { const percentiles = get ( aggregations , [ ... aggsPath , `${ safeFieldName }_percentiles` , 'values' ] , [ ] ) ; const medianPercentile = find ( percentiles , { key : 50 , } ) ; stats . median = medianPercentile !== undefined ? medianPercentile ! . value : 0 ; stats . distribution = this . processDistributionData ( percentiles , PERCENTILE_SPACING , stats . min ) ; } batchStats . push ( stats ) ; } ) ; return batchStats ; } async getStringFieldsStats ( indexPatternTitle , query , fields , samplerShardSize , timeFieldName , earliestMs , latestMs , runtimeMappings ? ) { const index = indexPatternTitle ; const size = 0 ; const filterCriteria = buildBaseFilterCriteria ( timeFieldName , earliestMs , latestMs , query ) ; const aggs = { } ; fields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field . fieldName , i ) ; const top = { terms : { field : field . fieldName , size : 10 , order : { _count : 'desc' , } , } , } ; if ( samplerShardSize < 1 && field . cardinality >= SAMPLER_TOP_TERMS_THRESHOLD ) { aggs [ `${ safeFieldName }_top` ] = { sampler : { shard_size : SAMPLER_TOP_TERMS_SHARD_SIZE , } , aggs : { top , } , } ; } else { aggs [ `${ safeFieldName }_top` ] = top ; } } ) ; const searchBody = { query : { bool : { filter : filterCriteria , } , } , aggs : buildSamplerAggregation ( aggs , samplerShardSize ) , ... ( isPopulatedObject ( runtimeMappings ) ? { runtime_mappings : runtimeMappings } : { } ) , } ; const { body } = await this . _asCurrentUser . search ( { index , size , body : searchBody , } ) ; const aggregations = body . aggregations ; const aggsPath = getSamplerAggregationsResponsePath ( samplerShardSize ) ; const batchStats = [ ] ; fields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field . fieldName , i ) ; const topAggsPath = [ ... aggsPath , `${ safeFieldName }_top` ] ; if ( samplerShardSize < 1 && field . cardinality >= SAMPLER_TOP_TERMS_THRESHOLD ) { topAggsPath . push ( 'top' ) ; } const topValues = get ( aggregations , [ ... topAggsPath , 'buckets' ] , [ ] ) ; const stats = { fieldName : field . fieldName , isTopValuesSampled : field . cardinality >= SAMPLER_TOP_TERMS_THRESHOLD || samplerShardSize > 0 , topValues , topValuesSampleSize : topValues . reduce ( ( acc , curr ) => acc + curr . doc_count , get ( aggregations , [ ... topAggsPath , 'sum_other_doc_count' ] , 0 ) ) , topValuesSamplerShardSize : field . cardinality >= SAMPLER_TOP_TERMS_THRESHOLD ? SAMPLER_TOP_TERMS_SHARD_SIZE : samplerShardSize , } ; batchStats . push ( stats ) ; } ) ; return batchStats ; } async getDateFieldsStats ( indexPatternTitle , query , fields , samplerShardSize , timeFieldName , earliestMs , latestMs , runtimeMappings ? ) { const index = indexPatternTitle ; const size = 0 ; const filterCriteria = buildBaseFilterCriteria ( timeFieldName , earliestMs , latestMs , query ) ; const aggs = { } ; fields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field . fieldName , i ) ; aggs [ `${ safeFieldName }_field_stats` ] = { filter : { exists : { field : field . fieldName } } , aggs : { actual_stats : { stats : { field : field . fieldName } , } , } , } ; } ) ; const searchBody = { query : { bool : { filter : filterCriteria , } , } , aggs : buildSamplerAggregation ( aggs , samplerShardSize ) , ... ( isPopulatedObject ( runtimeMappings ) ? { runtime_mappings : runtimeMappings } : { } ) , } ; const { body } = await this . _asCurrentUser . search ( { index , size , body : searchBody , } ) ; const aggregations = body . aggregations ; const aggsPath = getSamplerAggregationsResponsePath ( samplerShardSize ) ; const batchStats = [ ] ; fields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field . fieldName , i ) ; const docCount = get ( aggregations , [ ... aggsPath , `${ safeFieldName }_field_stats` , 'doc_count' ] , 0 ) ; const fieldStatsResp = get ( aggregations , [ ... aggsPath , `${ safeFieldName }_field_stats` , 'actual_stats' ] , { } ) ; batchStats . push ( { fieldName : field . fieldName , count : docCount , earliest : get ( fieldStatsResp , 'min' , 0 ) , latest : get ( fieldStatsResp , 'max' , 0 ) , } ) ; } ) ; return batchStats ; } async getBooleanFieldsStats ( indexPatternTitle , query , fields , samplerShardSize , timeFieldName , earliestMs , latestMs , runtimeMappings ? ) { const index = indexPatternTitle ; const size = 0 ; const filterCriteria = buildBaseFilterCriteria ( timeFieldName , earliestMs , latestMs , query ) ; const aggs = { } ; fields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field . fieldName , i ) ; aggs [ `${ safeFieldName }_value_count` ] = { filter : { exists : { field : field . fieldName } } , } ; aggs [ `${ safeFieldName }_values` ] = { terms : { field : field . fieldName , size : 2 , } , } ; } ) ; const searchBody = { query : { bool : { filter : filterCriteria , } , } , aggs : buildSamplerAggregation ( aggs , samplerShardSize ) , ... ( isPopulatedObject ( runtimeMappings ) ? { runtime_mappings : runtimeMappings } : { } ) , } ; const { body } = await this . _asCurrentUser . search ( { index , size , body : searchBody , } ) ; const aggregations = body . aggregations ; const aggsPath = getSamplerAggregationsResponsePath ( samplerShardSize ) ; const batchStats = [ ] ; fields . forEach ( ( field , i ) => { const safeFieldName = getSafeAggregationName ( field . fieldName , i ) ; const stats = { fieldName : field . fieldName , count : get ( aggregations , [ ... aggsPath , `${ safeFieldName }_value_count` , 'doc_count' ] , 0 ) , trueCount : 0 , falseCount : 0 , } ; const valueBuckets = get ( aggregations , [ ... aggsPath , `${ safeFieldName }_values` , 'buckets' ] , [ ] ) ; valueBuckets . forEach ( ( bucket ) => { stats [ `${ bucket . key_as_string }Count` ] = bucket . doc_count ; } ) ; batchStats . push ( stats ) ; } ) ; return batchStats ; } async getFieldExamples ( indexPatternTitle , query , field , timeFieldName , earliestMs , latestMs , maxExamples , runtimeMappings ? ) { const index = indexPatternTitle ; const size = Math . max ( 100 , maxExamples ) ; const filterCriteria = buildBaseFilterCriteria ( timeFieldName , earliestMs , latestMs , query ) ; filterCriteria . push ( { exists : { field } , } ) ; const searchBody = { fields : [ field ] , _source : false , query : { bool : { filter : filterCriteria , } , } , ... ( isPopulatedObject ( runtimeMappings ) ? { runtime_mappings : runtimeMappings } : { } ) , } ; const { body } = await this . _asCurrentUser . search ( { index , size , body : searchBody , } ) ; const stats = { fieldName : field , examples : [ ] as any [ ] , } ; if ( body . hits . total . value > 0 ) { const hits = body . hits . hits ; for ( let i = 0 ; i < hits . length ; i ++ ) { const doc = get ( hits [ i ] . fields , field ) ; if ( Array . isArray ( doc ) && doc . length > 0 ) { const example = doc [ 0 ] ; if ( example !== undefined && stats . examples . indexOf ( example ) === - 1 ) { stats . examples . push ( example ) ; if ( stats . examples . length === maxExamples ) { break ; } } } } } return stats ; } processDistributionData ( percentiles , percentileSpacing , minValue ) { const distribution = { percentiles : [ ] , minPercentile : 0 , maxPercentile : 100 } ; if ( percentiles . length === 0 ) { return distribution ; } let percentileBuckets = [ ] ; let lowerBound = minValue ; if ( lowerBound >= 0 ) { distribution . minPercentile = 0 ; distribution . maxPercentile = 90 ; percentileBuckets = percentiles . slice ( 0 , percentiles . length - 2 ) ; const lastValue = ( last ( percentileBuckets ) as any ) . value ; const upperBound = lowerBound + 1.5 * ( lastValue - lowerBound ) ; const filteredLength = percentileBuckets . length ; for ( let i = filteredLength ; i < percentiles . length ; i ++ ) { if ( percentiles [ i ] . value < upperBound ) { percentileBuckets . push ( percentiles [ i ] ) ; distribution . maxPercentile += percentileSpacing ; } else { break ; } } } else { const dataMin = lowerBound ; lowerBound = percentiles [ 0 ] . value ; distribution . minPercentile = 5 ; distribution . maxPercentile = 95 ; percentileBuckets = percentiles . slice ( 1 , percentiles . length - 1 ) ; const lastValue = ( last ( percentileBuckets ) as any ) . value ; const maxDiff = 0.25 * ( lastValue - lowerBound ) ; if ( lowerBound - dataMin < maxDiff ) { percentileBuckets . splice ( 0 , 0 , percentiles [ 0 ] ) ; distribution . minPercentile = 0 ; lowerBound = dataMin ; } if ( percentiles [ percentiles . length - 1 ] . value - lastValue < maxDiff ) { percentileBuckets . push ( percentiles [ percentiles . length - 1 ] ) ; distribution . maxPercentile = 100 ; } } const totalBuckets = percentileBuckets . length ; let lastBucketValue = lowerBound ; let numEqualValueBuckets = 0 ; for ( let i = 0 ; i < totalBuckets ; i ++ ) { const bucket = percentileBuckets [ i ] ; if ( bucket . value . toPrecision ( 15 ) !== lastBucketValue . toPrecision ( 15 ) ) { if ( numEqualValueBuckets > 0 ) { distribution . percentiles . push ( { percent : numEqualValueBuckets * percentileSpacing , minValue : lastBucketValue , maxValue : lastBucketValue , } ) ; } distribution . percentiles . push ( { percent : percentileSpacing , minValue : lastBucketValue , maxValue : bucket . value , } ) ; lastBucketValue = bucket . value ; numEqualValueBuckets = 0 ; } else { numEqualValueBuckets ++ ; if ( i === totalBuckets - 1 ) { distribution . percentiles . push ( { percent : numEqualValueBuckets * percentileSpacing , minValue : lastBucketValue , maxValue : lastBucketValue , } ) ; } } } return distribution ; } }