import { getNumTopClasses , getNumTopFeatureImportanceValues } from './analytics' ; import { Field } from '../../../../common/types/fields' ; import { getPredictedFieldName , getDependentVar , getPredictionFieldName , isClassificationAnalysis , isOutlierAnalysis , isRegressionAnalysis , } from '../../../../common/util/analytics_utils' ; import { ES_FIELD_TYPES , KBN_FIELD_TYPES } from '../../../../../../../src/plugins/data/public' ; import { newJobCapsServiceAnalytics } from '../../services/new_job_capabilities/new_job_capabilities_service_analytics' ; import { FEATURE_IMPORTANCE , FEATURE_INFLUENCE , OUTLIER_SCORE , TOP_CLASSES } from './constants' ; import { DataFrameAnalyticsConfig } from '../../../../common/types/data_frame_analytics' ; export type EsId = string ; export type EsDocSource = Record < string , any > ; export type EsFieldName = string ; export interface EsDoc extends Record < string , any > { _id : EsId ; _source : EsDocSource ; } export const MAX_COLUMNS = 10 ; export const DEFAULT_REGRESSION_COLUMNS = 8 ; export const BASIC_NUMERICAL_TYPES = new Set ( [ ES_FIELD_TYPES . UNSIGNED_LONG , ES_FIELD_TYPES . LONG , ES_FIELD_TYPES . INTEGER , ES_FIELD_TYPES . SHORT , ES_FIELD_TYPES . BYTE , ] ) ; export const EXTENDED_NUMERICAL_TYPES = new Set ( [ ES_FIELD_TYPES . DOUBLE , ES_FIELD_TYPES . FLOAT , ES_FIELD_TYPES . HALF_FLOAT , ES_FIELD_TYPES . SCALED_FLOAT , ] ) ; export const ML__ID_COPY = 'ml__id_copy' ; export const ML__INCREMENTAL_ID = 'ml__incremental_id' ; export const isKeywordAndTextType = ( fieldName ) : boolean => { const { fields } = newJobCapsServiceAnalytics ; const fieldType = fields . find ( ( field ) => field . name === fieldName ) ?. type ; let isBothTypes = false ; if ( fieldType !== undefined && fieldType === ES_FIELD_TYPES . KEYWORD ) { const field = newJobCapsServiceAnalytics . getFieldById ( fieldName . replace ( /\.keyword$/ , '' ) ) ; isBothTypes = field !== null && field . type === ES_FIELD_TYPES . TEXT ; } else if ( fieldType !== undefined && fieldType === ES_FIELD_TYPES . TEXT ) { const field = newJobCapsServiceAnalytics . getFieldById ( `${ fieldName }.keyword` ) ; isBothTypes = field !== null && field . type === ES_FIELD_TYPES . KEYWORD ; } return isBothTypes ; } ; export const sortExplorationResultsFields = ( a , b , jobConfig ) => { const resultsField = jobConfig . dest . results_field ; if ( isOutlierAnalysis ( jobConfig . analysis ) ) { if ( a === `${ resultsField }.${ OUTLIER_SCORE }` ) { return - 1 ; } if ( b === `${ resultsField }.${ OUTLIER_SCORE }` ) { return 1 ; } } if ( isClassificationAnalysis ( jobConfig . analysis ) || isRegressionAnalysis ( jobConfig . analysis ) ) { const dependentVariable = getDependentVar ( jobConfig . analysis ) ; const predictedField = getPredictedFieldName ( resultsField , jobConfig . analysis , true ) ; if ( a === `${ resultsField }.is_training` ) { return - 1 ; } if ( b === `${ resultsField }.is_training` ) { return 1 ; } if ( a === predictedField ) { return - 1 ; } if ( b === predictedField ) { return 1 ; } if ( a === dependentVariable || a === dependentVariable . replace ( /\.keyword$/ , '' ) ) { return - 1 ; } if ( b === dependentVariable || b === dependentVariable . replace ( /\.keyword$/ , '' ) ) { return 1 ; } if ( a === `${ resultsField }.prediction_probability` ) { return - 1 ; } if ( b === `${ resultsField }.prediction_probability` ) { return 1 ; } } const typeofA = typeof a ; const typeofB = typeof b ; const tokensA = a . split ( '.' ) ; const prefixA = tokensA [ 0 ] ; const tokensB = b . split ( '.' ) ; const prefixB = tokensB [ 0 ] ; if ( prefixA === resultsField && tokensA . length > 1 && prefixB !== resultsField ) { tokensA . shift ( ) ; tokensA . shift ( ) ; if ( tokensA . join ( '.' ) === b ) return 1 ; return tokensA . join ( '.' ) . localeCompare ( b ) ; } if ( prefixB === resultsField && tokensB . length > 1 && prefixA !== resultsField ) { tokensB . shift ( ) ; tokensB . shift ( ) ; if ( tokensB . join ( '.' ) === a ) return - 1 ; return a . localeCompare ( tokensB . join ( '.' ) ) ; } if ( typeofA !== 'string' && typeofB === 'string' ) { return 1 ; } if ( typeofA === 'string' && typeofB !== 'string' ) { return - 1 ; } if ( typeofA === 'string' && typeofB === 'string' ) { return a . localeCompare ( b ) ; } return a . localeCompare ( b ) ; } ; export const getDefaultFieldsFromJobCaps = ( fields , jobConfig , needsDestIndexFields ) : { selectedFields : Field [ ] ; docFields : Field [ ] ; depVarType ? : ES_FIELD_TYPES ; } => { const fieldsObj = { selectedFields : [ ] , docFields : [ ] , } ; if ( fields . length === 0 ) { return fieldsObj ; } const resultsField = jobConfig . dest . results_field ; const allFields = [ ] ; let type ; let predictedField ; if ( isOutlierAnalysis ( jobConfig . analysis ) ) { if ( ! jobConfig . analysis . outlier_detection . compute_feature_influence ) { fields = fields . filter ( ( field ) => ! field . name . includes ( `${ resultsField }.${ FEATURE_INFLUENCE }` ) ) ; } else { fields = fields . filter ( ( field ) => ! field . name . includes ( `${ resultsField }.${ FEATURE_INFLUENCE }.` ) ) ; } if ( needsDestIndexFields === true ) { allFields . push ( { id : `${ resultsField }.${ OUTLIER_SCORE }` , name : `${ resultsField }.${ OUTLIER_SCORE }` , type : KBN_FIELD_TYPES . NUMBER , } ) ; } } if ( isClassificationAnalysis ( jobConfig . analysis ) || isRegressionAnalysis ( jobConfig . analysis ) ) { const dependentVariable = getDependentVar ( jobConfig . analysis ) ; type = newJobCapsServiceAnalytics . getFieldById ( dependentVariable ) ?. type ; const predictionFieldName = getPredictionFieldName ( jobConfig . analysis ) ; const numTopFeatureImportanceValues = getNumTopFeatureImportanceValues ( jobConfig . analysis ) ; const numTopClasses = getNumTopClasses ( jobConfig . analysis ) ; const defaultPredictionField = `${ dependentVariable }_prediction` ; predictedField = `${ resultsField }.${ predictionFieldName ? predictionFieldName : defaultPredictionField }` ; if ( ( numTopFeatureImportanceValues ?? 0 ) === 0 ) { fields = fields . filter ( ( field ) => ! field . name . includes ( `${ resultsField }.${ FEATURE_IMPORTANCE }` ) ) ; } else { fields = fields . filter ( ( field ) => ! field . name . includes ( `${ resultsField }.${ FEATURE_IMPORTANCE }.` ) ) ; } if ( ( numTopClasses ?? 0 ) === 0 ) { fields = fields . filter ( ( field ) => ! field . name . includes ( `${ resultsField }.${ TOP_CLASSES }` ) ) ; } else { fields = fields . filter ( ( field ) => ! field . name . includes ( `${ resultsField }.${ TOP_CLASSES }.` ) ) ; } if ( needsDestIndexFields === true ) { allFields . push ( { id : `${ resultsField }.is_training` , name : `${ resultsField }.is_training` , type : ES_FIELD_TYPES . BOOLEAN , } , { id : predictedField , name : predictedField , type } ) ; } } allFields . push ( ... fields ) ; allFields . sort ( ( { name : a } , { name : b } ) => sortExplorationResultsFields ( a , b , jobConfig ) ) ; let selectedFields = allFields . filter ( ( field ) => field . name === predictedField || ! field . name . includes ( '.keyword' ) ) ; if ( selectedFields . length > DEFAULT_REGRESSION_COLUMNS ) { selectedFields = selectedFields . slice ( 0 , DEFAULT_REGRESSION_COLUMNS ) ; } return { selectedFields , docFields : allFields , depVarType : type , } ; } ;