import { keyBy , last } from 'lodash' ; import { Logger } from 'kibana/server' ; import util from 'util' ; import { maybe } from '../../../../common/utils/maybe' ; import { ProfileStackFrame } from '../../../../typings/es_schemas/ui/profile' ; import { ProfilingValueType , ProfileNode , getValueTypeConfig , } from '../../../../common/profiling' ; import { ProcessorEvent } from '../../../../common/processor_event' ; import { ESFilter } from '../../../../../../../src/core/types/elasticsearch' ; import { PROFILE_STACK , PROFILE_TOP_ID , SERVICE_NAME , } from '../../../../common/elasticsearch_fieldnames' ; import { rangeQuery , kqlQuery } from '../../../../../observability/server' ; import { environmentQuery } from '../../../../common/utils/environment_query' ; import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client' ; import { Setup } from '../../../lib/helpers/setup_request' ; import { withApmSpan } from '../../../utils/with_apm_span' ; const MAX_STACK_IDS = 10000 ; const MAX_STACKS_PER_REQUEST = 1000 ; const maybeAdd = ( to , value ) => { if ( to . includes ( value ) ) { return ; } to . push ( value ) ; } ; async function getProfilingStats ( { apmEventClient , filter , valueTypeField , } ) { const response = await apmEventClient . search ( 'get_profiling_stats' , { apm : { events : [ ProcessorEvent . profile ] , } , body : { size : 0 , query : { bool : { filter , } , } , aggs : { stacks : { terms : { field : PROFILE_TOP_ID , size : MAX_STACK_IDS , order : { value : 'desc' , } , } , aggs : { value : { sum : { field : valueTypeField , } , } , } , } , } , } , } ) ; const stacks = response . aggregations ?. stacks . buckets . map ( ( stack ) => { return { id : stack . key as string , value : stack . value . value ! , } ; } ) ?? [ ] ; return stacks ; } function getProfilesWithStacks ( { apmEventClient , filter , } ) { return withApmSpan ( 'get_profiles_with_stacks' , async ( ) => { const cardinalityResponse = await apmEventClient . search ( 'get_top_cardinality' , { apm : { events : [ ProcessorEvent . profile ] , } , body : { size : 0 , query : { bool : { filter } , } , aggs : { top : { cardinality : { field : PROFILE_TOP_ID , } , } , } , } , } ) ; const cardinality = cardinalityResponse . aggregations ?. top . value ?? 0 ; const numStacksToFetch = Math . min ( Math . ceil ( cardinality * 1.1 ) , MAX_STACK_IDS ) ; const partitions = Math . ceil ( numStacksToFetch / MAX_STACKS_PER_REQUEST ) ; if ( partitions === 0 ) { return [ ] ; } const allResponses = await withApmSpan ( 'get_all_stacks' , async ( ) => { return Promise . all ( [ ... new Array ( partitions ) ] . map ( async ( _ , num ) => { const response = await apmEventClient . search ( 'get_partition' , { apm : { events : [ ProcessorEvent . profile ] , } , body : { query : { bool : { filter , } , } , aggs : { top : { terms : { field : PROFILE_TOP_ID , size : Math . max ( MAX_STACKS_PER_REQUEST ) , include : { num_partitions : partitions , partition : num , } , } , aggs : { latest : { top_hits : { _source : [ PROFILE_TOP_ID , PROFILE_STACK ] , } , } , } , } , } , } , } ) ; return ( response . aggregations ?. top . buckets . flatMap ( ( bucket ) => { return bucket . latest . hits . hits [ 0 ] . _source ; } ) ?? [ ] ) ; } ) ) ; } ) ; return allResponses . flat ( ) ; } ) ; } export async function getServiceProfilingStatistics ( { kuery , serviceName , setup , environment , valueType , logger , start , end , } ) { return withApmSpan ( 'get_service_profiling_statistics' , async ( ) => { const { apmEventClient } = setup ; const valueTypeField = getValueTypeConfig ( valueType ) . field ; const filter = [ { term : { [ SERVICE_NAME ] : serviceName } } , { exists : { field : valueTypeField } } , ... rangeQuery ( start , end ) , ... environmentQuery ( environment ) , ... kqlQuery ( kuery ) , ] ; const [ profileStats , profileStacks ] = await Promise . all ( [ getProfilingStats ( { apmEventClient , filter , valueTypeField } ) , getProfilesWithStacks ( { apmEventClient , filter } ) , ] ) ; const nodes = { } ; const rootNodes = [ ] ; function getNode ( frame ) { const { id , filename , function : functionName , line } = frame ; const location = [ functionName , line ] . filter ( Boolean ) . join ( ':' ) ; const fqn = [ filename , location ] . filter ( Boolean ) . join ( '/' ) ; const label = last ( location . split ( '/' ) ) ! ; let node = nodes [ id ] ; if ( ! node ) { node = { id , label , fqn , value : 0 , children : [ ] } ; nodes [ id ] = node ; } return node ; } const stackStatsById = keyBy ( profileStats , 'id' ) ; const missingStacks = [ ] ; profileStacks . forEach ( ( profile ) => { const stats = maybe ( stackStatsById [ profile . profile . top . id ] ) ; if ( ! stats ) { missingStacks . push ( profile . profile . top . id ) ; return ; } const frames = profile . profile . stack . concat ( ) . reverse ( ) ; frames . forEach ( ( frame , index ) => { const node = getNode ( frame ) ; if ( index === frames . length - 1 && stats ) { node . value += stats . value ; } if ( index === 0 ) { maybeAdd ( rootNodes , node . id ) ; } else { const parent = nodes [ frames [ index - 1 ] . id ] ; maybeAdd ( parent . children , node . id ) ; } } ) ; } ) ; if ( missingStacks . length > 0 ) { logger . warn ( `Could not find stats for all stacks: ${ util . inspect ( { numProfileStats : profileStats . length , numStacks : profileStacks . length , missing : missingStacks , } ) }` ) ; } return { nodes , rootNodes , } ; } ) ; }