import Boom from '@hapi/boom' ; import uuidv5 from 'uuid/v5' ; import { set } from '@elastic/safer-lodash-set' ; import _ from 'lodash' ; import Semver from 'semver' ; import { Logger } from '../../../logging' ; import { SavedObjectUnsanitizedDoc } from '../../serialization' ; import { SavedObjectsMigrationVersion , SavedObjectsNamespaceType , SavedObjectsType , } from '../../types' ; import { MigrationLogger } from './migration_logger' ; import { TransformSavedObjectDocumentError } from '.' ; import { ISavedObjectTypeRegistry } from '../../saved_objects_type_registry' ; import { SavedObjectMigrationFn , SavedObjectMigrationMap } from '../types' ; import { DEFAULT_NAMESPACE_STRING , SavedObjectsUtils } from '../../service/lib/utils' ; import { LegacyUrlAlias , LEGACY_URL_ALIAS_TYPE } from '../../object_types' ; const DEFAULT_MINIMUM_CONVERT_VERSION = '8.0.0' ; export type MigrateFn = ( doc : SavedObjectUnsanitizedDoc ) => SavedObjectUnsanitizedDoc ; export type MigrateAndConvertFn = ( doc : SavedObjectUnsanitizedDoc ) => SavedObjectUnsanitizedDoc [ ] ; interface TransformResult { /**
     * This is the original document that has been transformed.
     */ transformedDoc : SavedObjectUnsanitizedDoc ; /**
     * These are any new document(s) that have been created during the transformation process; these are not transformed, but they are marked
     * as up-to-date. Only conversion transforms generate additional documents.
     */ additionalDocs : SavedObjectUnsanitizedDoc [ ] ; } type ApplyTransformsFn = ( doc : SavedObjectUnsanitizedDoc , options ? : TransformOptions ) => TransformResult ; interface TransformOptions { convertNamespaceTypes ? : boolean ; } interface DocumentMigratorOptions { kibanaVersion : string ; typeRegistry : ISavedObjectTypeRegistry ; minimumConvertVersion ? : string ; log : Logger ; } interface ActiveMigrations { [ type ] : { /** Derived from `migrate` transforms and `convert` transforms */ latestMigrationVersion ? : string ; /** Derived from `reference` transforms */ latestCoreMigrationVersion ? : string ; transforms : Transform [ ] ; } ; } interface Transform { version : string ; transform : ( doc : SavedObjectUnsanitizedDoc ) => TransformResult ; /**
     * There are two "migrationVersion" transform types:
     *   * `migrate` - These transforms are defined and added by consumers using the type registry; each is applied to a single object type
     *     based on an object's `migrationVersion[type]` field. These are applied during index migrations and document migrations.
     *   * `convert` - These transforms are defined by core and added by consumers using the type registry; each is applied to a single object
     *     type based on an object's `migrationVersion[type]` field. These are applied during index migrations, NOT document migrations.
     *
     * There is one "coreMigrationVersion" transform type:
     *   * `reference` - These transforms are defined by core and added by consumers using the type registry; they are applied to all object
     *     types based on their `coreMigrationVersion` field. These are applied during index migrations, NOT document migrations.
     *
     * If any additional transform types are added, the functions below should be updated to account for them.
     */ transformType : 'migrate' | 'convert' | 'reference' ; } /**
 * Manages migration of individual documents.
 */ export interface VersionedTransformer { migrationVersion : SavedObjectsMigrationVersion ; migrate : MigrateFn ; migrateAndConvert : MigrateAndConvertFn ; prepareMigrations : ( ) => void ; } /**
 * A concrete implementation of the VersionedTransformer interface.
 */ export class DocumentMigrator implements VersionedTransformer { private documentMigratorOptions : Omit < DocumentMigratorOptions , 'minimumConvertVersion' > ; private migrations ? : ActiveMigrations ; private transformDoc ? : ApplyTransformsFn ; param { DocumentMigratorOptions } opts prop prop prop prop memberof constructor ( { typeRegistry , kibanaVersion , minimumConvertVersion = DEFAULT_MINIMUM_CONVERT_VERSION , log , } ) { validateMigrationDefinition ( typeRegistry , kibanaVersion , minimumConvertVersion ) ; this . documentMigratorOptions = { typeRegistry , kibanaVersion , log } ; } readonly type { SavedObjectsMigrationVersion } memberof public get migrationVersion ( ) : SavedObjectsMigrationVersion { if ( ! this . migrations ) { throw new Error ( 'Migrations are not ready. Make sure prepareMigrations is called first.' ) ; } return Object . entries ( this . migrations ) . reduce ( ( acc , [ prop , { latestMigrationVersion } ] ) => { if ( latestMigrationVersion ) { return { ... acc , [ prop ] : latestMigrationVersion } ; } return acc ; } , { } ) ; } returns { void } memberof public prepareMigrations = ( ) => { const { typeRegistry , kibanaVersion , log } = this . documentMigratorOptions ; this . migrations = buildActiveMigrations ( typeRegistry , kibanaVersion , log ) ; this . transformDoc = buildDocumentTransform ( { kibanaVersion , migrations : this . migrations , } ) ; } ; param { SavedObjectUnsanitizedDoc } doc returns { SavedObjectUnsanitizedDoc } memberof public migrate = ( doc ) : SavedObjectUnsanitizedDoc => { if ( ! this . migrations || ! this . transformDoc ) { throw new Error ( 'Migrations are not ready. Make sure prepareMigrations is called first.' ) ; } const clonedDoc = _ . cloneDeep ( doc ) ; const { transformedDoc } = this . transformDoc ( clonedDoc ) ; return transformedDoc ; } ; param { SavedObjectUnsanitizedDoc } doc returns { SavedObjectUnsanitizedDoc } memberof public migrateAndConvert = ( doc ) : SavedObjectUnsanitizedDoc [ ] => { if ( ! this . migrations || ! this . transformDoc ) { throw new Error ( 'Migrations are not ready. Make sure prepareMigrations is called first.' ) ; } const clonedDoc = _ . cloneDeep ( doc ) ; const { transformedDoc , additionalDocs } = this . transformDoc ( clonedDoc , { convertNamespaceTypes : true , } ) ; return [ transformedDoc , ... additionalDocs ] ; } ; } function validateMigrationsMapObject ( name , kibanaVersion , migrationsMap ? ) { function assertObject ( obj , prefix ) { if ( ! obj || typeof obj !== 'object' ) { throw new Error ( `${ prefix } Got ${ obj }.` ) ; } } function assertValidSemver ( version , type ) { if ( ! Semver . valid ( version ) ) { throw new Error ( `Invalid migration for type ${ type }. Expected all properties to be semvers, but got ${ version }.` ) ; } if ( Semver . gt ( version , kibanaVersion ) ) { throw new Error ( `Invalid migration for type ${ type }. Property '${ version }' cannot be greater than the current Kibana version '${ kibanaVersion }'.` ) ; } } function assertValidTransform ( fn , version , type ) { if ( typeof fn !== 'function' ) { throw new Error ( `Invalid migration ${ type }.${ version }: expected a function, but got ${ fn }.` ) ; } } if ( migrationsMap ) { assertObject ( migrationsMap , `Migrations map for type ${ name } should be an object like { '2.0.0': (doc) => doc }.` ) ; Object . entries ( migrationsMap ) . forEach ( ( [ version , fn ] ) => { assertValidSemver ( version , name ) ; assertValidTransform ( fn , version , name ) ; } ) ; } } /**
 * Basic validation that the migration definition matches our expectations. We can't
 * rely on TypeScript here, as the caller may be JavaScript / ClojureScript / any compile-to-js
 * language. So, this is just to provide a little developer-friendly error messaging. Joi was
 * giving weird errors, so we're just doing manual validation.
 */ function validateMigrationDefinition ( registry , kibanaVersion , minimumConvertVersion ) { function assertObjectOrFunction ( entity , prefix ) { if ( ! entity || ( typeof entity !== 'function' && typeof entity !== 'object' ) ) { throw new Error ( `${ prefix } Got! ${ typeof entity }, ${ JSON . stringify ( entity ) }.` ) ; } } function assertValidConvertToMultiNamespaceType ( namespaceType , convertToMultiNamespaceTypeVersion , type ) { if ( namespaceType !== 'multiple' && namespaceType !== 'multiple-isolated' ) { throw new Error ( `Invalid convertToMultiNamespaceTypeVersion for type ${ type }. Expected namespaceType to be 'multiple' or 'multiple-isolated', but got '${ namespaceType }'.` ) ; } else if ( ! Semver . valid ( convertToMultiNamespaceTypeVersion ) ) { throw new Error ( `Invalid convertToMultiNamespaceTypeVersion for type ${ type }. Expected value to be a semver, but got '${ convertToMultiNamespaceTypeVersion }'.` ) ; } else if ( Semver . lt ( convertToMultiNamespaceTypeVersion , minimumConvertVersion ) ) { throw new Error ( `Invalid convertToMultiNamespaceTypeVersion for type ${ type }. Value '${ convertToMultiNamespaceTypeVersion }' cannot be less than '${ minimumConvertVersion }'.` ) ; } else if ( Semver . gt ( convertToMultiNamespaceTypeVersion , kibanaVersion ) ) { throw new Error ( `Invalid convertToMultiNamespaceTypeVersion for type ${ type }. Value '${ convertToMultiNamespaceTypeVersion }' cannot be greater than the current Kibana version '${ kibanaVersion }'.` ) ; } else if ( Semver . patch ( convertToMultiNamespaceTypeVersion ) ) { throw new Error ( `Invalid convertToMultiNamespaceTypeVersion for type ${ type }. Value '${ convertToMultiNamespaceTypeVersion }' cannot be used on a patch version (must be like 'x.y.0').` ) ; } } registry . getAllTypes ( ) . forEach ( ( type ) => { const { name , migrations , convertToMultiNamespaceTypeVersion , namespaceType } = type ; if ( migrations ) { assertObjectOrFunction ( type . migrations , `Migration for type ${ name } should be an object or a function returning an object like { '2.0.0': (doc) => doc }.` ) ; } if ( convertToMultiNamespaceTypeVersion ) { assertValidConvertToMultiNamespaceType ( namespaceType , convertToMultiNamespaceTypeVersion , name ) ; } } ) ; } /**
 * Converts migrations from a format that is convenient for callers to a format that
 * is convenient for our internal usage:
 * From: { type: { version: fn } }
 * To:   { type: { latestMigrationVersion?: string; latestCoreMigrationVersion?: string; transforms: [{ version: string, transform: fn }] } }
 */ function buildActiveMigrations ( typeRegistry , kibanaVersion , log ) { const referenceTransforms = getReferenceTransforms ( typeRegistry ) ; return typeRegistry . getAllTypes ( ) . reduce ( ( migrations , type ) => { const migrationsMap = typeof type . migrations === 'function' ? type . migrations ( ) : type . migrations ; validateMigrationsMapObject ( type . name , kibanaVersion , migrationsMap ) ; const migrationTransforms = Object . entries ( migrationsMap ?? { } ) . map < Transform > ( ( [ version , transform ] ) => ( { version , transform : wrapWithTry ( version , type , transform , log ) , transformType : 'migrate' , } ) ) ; const conversionTransforms = getConversionTransforms ( type ) ; const transforms = [ ... referenceTransforms , ... conversionTransforms , ... migrationTransforms , ] . sort ( transformComparator ) ; if ( ! transforms . length ) { return migrations ; } const migrationVersionTransforms = [ ] ; const coreMigrationVersionTransforms = [ ] ; transforms . forEach ( ( x ) => { if ( x . transformType === 'migrate' || x . transformType === 'convert' ) { migrationVersionTransforms . push ( x ) ; } else { coreMigrationVersionTransforms . push ( x ) ; } } ) ; return { ... migrations , [ type . name ] : { latestMigrationVersion : _ . last ( migrationVersionTransforms ) ?. version , latestCoreMigrationVersion : _ . last ( coreMigrationVersionTransforms ) ?. version , transforms , } , } ; } , { } as ActiveMigrations ) ; } /**
 * Creates a function which migrates and validates any document that is passed to it.
 */ function buildDocumentTransform ( { kibanaVersion , migrations , } ) { return function transformAndValidate ( doc , options = { } ) { validateCoreMigrationVersion ( doc , kibanaVersion ) ; const { convertNamespaceTypes = false } = options ; let transformedDoc ; let additionalDocs = [ ] ; if ( doc . migrationVersion ) { const result = applyMigrations ( doc , migrations , kibanaVersion , convertNamespaceTypes ) ; transformedDoc = result . transformedDoc ; additionalDocs = additionalDocs . concat ( result . additionalDocs . map ( ( x ) => markAsUpToDate ( x , migrations , kibanaVersion ) ) ) ; } else { transformedDoc = markAsUpToDate ( doc , migrations , kibanaVersion ) ; } if ( _ . isEmpty ( transformedDoc . migrationVersion ) ) { delete transformedDoc . migrationVersion ; } return { transformedDoc , additionalDocs } ; } ; } function validateCoreMigrationVersion ( doc , kibanaVersion ) { const { id , coreMigrationVersion : docVersion } = doc ; if ( ! docVersion ) { return ; } if ( ! Semver . valid ( docVersion ) ) { throw Boom . badData ( `Document "${ id }" has an invalid "coreMigrationVersion" [${ docVersion }]. This must be a semver value.` , doc ) ; } if ( doc . coreMigrationVersion && Semver . gt ( docVersion , kibanaVersion ) ) { throw Boom . badData ( `Document "${ id }" has a "coreMigrationVersion" which belongs to a more recent version` + ` of Kibana [${ docVersion }]. The current version is [${ kibanaVersion }].` , doc ) ; } } function applyMigrations ( doc , migrations , kibanaVersion , convertNamespaceTypes ) { let additionalDocs = [ ] ; while ( true ) { const prop = nextUnmigratedProp ( doc , migrations ) ; if ( ! prop ) { return { transformedDoc : { ... doc , coreMigrationVersion : kibanaVersion } , additionalDocs , } ; } const result = migrateProp ( doc , prop , migrations , convertNamespaceTypes ) ; doc = result . transformedDoc ; additionalDocs = [ ... additionalDocs , ... result . additionalDocs ] ; } } /**
 * Gets the doc's props, handling the special case of "type".
 */ function props ( doc ) { return Object . keys ( doc ) . concat ( doc . type ) ; } /**
 * Looks up the prop version in a saved object document or in our latest migrations.
 */ function propVersion ( doc , prop ) { return ( ( ( doc as any ) [ prop ] && ( doc as any ) [ prop ] . latestMigrationVersion ) || ( doc . migrationVersion && ( doc as any ) . migrationVersion [ prop ] ) ) ; } /**
 * Sets the doc's migrationVersion to be the most recent version
 */ function markAsUpToDate ( doc , migrations , kibanaVersion ) { return { ... doc , migrationVersion : props ( doc ) . reduce ( ( acc , prop ) => { const version = propVersion ( migrations , prop ) ; return version ? set ( acc , prop , version ) : acc ; } , { } ) , coreMigrationVersion : kibanaVersion , } ; } /**
 * Converts a single-namespace object to a multi-namespace object. This primarily entails removing the `namespace` field and adding the
 * `namespaces` field.
 *
 * If the object does not exist in the default namespace (undefined), its ID is also regenerated, and an "originId" is added to preserve
 * legacy import/copy behavior.
 */ function convertNamespaceType ( doc ) { const { namespace , ... otherAttrs } = doc ; const additionalDocs = [ ] ; if ( namespace === undefined ) { return { transformedDoc : { ... otherAttrs , namespaces : [ DEFAULT_NAMESPACE_STRING ] } , additionalDocs , } ; } const { id : originId , type } = otherAttrs ; const id = SavedObjectsUtils . getConvertedObjectId ( namespace , type , originId ! ) ; if ( namespace !== undefined ) { const legacyUrlAlias = { id : `${ namespace }:${ type }:${ originId }` , type : LEGACY_URL_ALIAS_TYPE , attributes : { sourceId : originId , targetNamespace : namespace , targetType : type , targetId : id , } , } ; additionalDocs . push ( legacyUrlAlias ) ; } return { transformedDoc : { ... otherAttrs , id , originId , namespaces : [ namespace ] } , additionalDocs , } ; } /**
 * Returns all applicable conversion transforms for a given object type.
 */ function getConversionTransforms ( type ) { const { convertToMultiNamespaceTypeVersion } = type ; if ( ! convertToMultiNamespaceTypeVersion ) { return [ ] ; } return [ { version : convertToMultiNamespaceTypeVersion , transform : convertNamespaceType , transformType : 'convert' , } , ] ; } /**
 * Returns all applicable reference transforms for all object types.
 */ function getReferenceTransforms ( typeRegistry ) { const transformMap = typeRegistry . getAllTypes ( ) . filter ( ( type ) => type . convertToMultiNamespaceTypeVersion ) . reduce ( ( acc , { convertToMultiNamespaceTypeVersion : version , name } ) => { const types = acc . get ( version ! ) ?? new Set ( ) ; return acc . set ( version ! , types . add ( name ) ) ; } , new Map < string , Set < string > > ( ) ) ; return Array . from ( transformMap , ( [ version , types ] ) => ( { version , transform : ( doc ) => { const { namespace , references } = doc ; if ( namespace && references ?. length ) { return { transformedDoc : { ... doc , references : references . map ( ( { type , id , ... attrs } ) => ( { ... attrs , type , id : types . has ( type ) ? SavedObjectsUtils . getConvertedObjectId ( namespace , type , id ) : id , } ) ) , } , additionalDocs : [ ] , } ; } return { transformedDoc : doc , additionalDocs : [ ] } ; } , transformType : 'reference' , } ) ) ; } /**
 * Transforms are sorted in ascending order by version. One version may contain multiple transforms; 'reference' transforms always run
 * first, 'convert' transforms always run second, and 'migrate' transforms always run last. This is because:
 *  1. 'convert' transforms get rid of the `namespace` field, which must be present for 'reference' transforms to function correctly.
 *  2. 'migrate' transforms are defined by the consumer, and may change the object type or migrationVersion which resets the migration loop
 *     and could cause any remaining transforms for this version to be skipped.
 */ function transformComparator ( a , b ) { const semver = Semver . compare ( a . version , b . version ) ; if ( semver !== 0 ) { return semver ; } else if ( a . transformType !== b . transformType ) { if ( a . transformType === 'migrate' ) { return 1 ; } else if ( b . transformType === 'migrate' ) { return - 1 ; } else if ( a . transformType === 'convert' ) { return 1 ; } else if ( b . transformType === 'convert' ) { return - 1 ; } } return 0 ; } /**
 * If a specific transform function fails, this tacks on a bit of information
 * about the document and transform that caused the failure.
 */ function wrapWithTry ( version , type , migrationFn , log ) { const context = Object . freeze ( { log : new MigrationLogger ( log ) , migrationVersion : version , convertToMultiNamespaceTypeVersion : type . convertToMultiNamespaceTypeVersion , isSingleNamespaceType : type . namespaceType === 'single' , } ) ; return function tryTransformDoc ( doc ) { try { const result = migrationFn ( doc , context ) ; if ( ! result || ! result . type ) { throw new Error ( `Invalid saved object returned from migration ${ type . name }:${ version }.` ) ; } return { transformedDoc : result , additionalDocs : [ ] } ; } catch ( error ) { log . error ( error ) ; throw new TransformSavedObjectDocumentError ( error , version ) ; } } ; } /**
 * Determines whether or not a document has any pending transforms that should be applied based on its coreMigrationVersion field.
 * Currently, only reference transforms qualify.
 */ function getHasPendingCoreMigrationVersionTransform ( doc , migrations , prop ) { if ( ! migrations . hasOwnProperty ( prop ) ) { return false ; } const { latestCoreMigrationVersion } = migrations [ prop ] ; const { coreMigrationVersion } = doc ; return ( latestCoreMigrationVersion && ( ! coreMigrationVersion || Semver . gt ( latestCoreMigrationVersion , coreMigrationVersion ) ) ) ; } /**
 * Finds the first unmigrated property in the specified document.
 */ function nextUnmigratedProp ( doc , migrations ) { return props ( doc ) . find ( ( p ) => { const latestMigrationVersion = propVersion ( migrations , p ) ; const docVersion = propVersion ( doc , p ) ; if ( docVersion && ( ! latestMigrationVersion || Semver . gt ( docVersion , latestMigrationVersion ) ) ) { throw Boom . badData ( `Document "${ doc . id }" has property "${ p }" which belongs to a more recent` + ` version of Kibana [${ docVersion }]. The last known version is [${ latestMigrationVersion }]` , doc ) ; } return ( ( latestMigrationVersion && latestMigrationVersion !== docVersion ) || getHasPendingCoreMigrationVersionTransform ( doc , migrations , p ) ) ; } ) ; } /**
 * Applies any relevant migrations to the document for the specified property.
 */ function migrateProp ( doc , prop , migrations , convertNamespaceTypes ) { const originalType = doc . type ; let migrationVersion = _ . clone ( doc . migrationVersion ) || { } ; let additionalDocs = [ ] ; for ( const { version , transform , transformType } of applicableTransforms ( migrations , doc , prop ) ) { if ( convertNamespaceTypes || ( transformType !== 'convert' && transformType !== 'reference' ) ) { const result = transform ( doc ) ; doc = result . transformedDoc ; additionalDocs = [ ... additionalDocs , ... result . additionalDocs ] ; } if ( transformType === 'reference' ) { doc . coreMigrationVersion = version ; } else { migrationVersion = updateMigrationVersion ( doc , migrationVersion , prop , version ) ; doc . migrationVersion = _ . clone ( migrationVersion ) ; } if ( doc . type !== originalType ) { break ; } } return { transformedDoc : doc , additionalDocs } ; } /**
 * Retrieves any prop transforms that have not been applied to doc.
 */ function applicableTransforms ( migrations , doc , prop ) { const minVersion = propVersion ( doc , prop ) ; const minReferenceVersion = doc . coreMigrationVersion || '0.0.0' ; const { transforms } = migrations [ prop ] ; return minVersion ? transforms . filter ( ( { version , transformType } ) => transformType === 'reference' ? Semver . gt ( version , minReferenceVersion ) : Semver . gt ( version , minVersion ) ) : transforms ; } /**
 * Updates the document's migrationVersion, ensuring that the calling transform
 * has not mutated migrationVersion in an unsupported way.
 */ function updateMigrationVersion ( doc , migrationVersion , prop , version ) { assertNoDowngrades ( doc , migrationVersion , prop , version ) ; const docVersion = propVersion ( doc , prop ) || '0.0.0' ; const maxVersion = Semver . gt ( docVersion , version ) ? docVersion : version ; return { ... ( doc . migrationVersion || migrationVersion ) , [ prop ] : maxVersion } ; } /**
 * Transforms that remove or downgrade migrationVersion properties are not allowed,
 * as this could get us into an infinite loop. So, we explicitly check for that here.
 */ function assertNoDowngrades ( doc , migrationVersion , prop , version ) { const docVersion = doc . migrationVersion ; if ( ! docVersion ) { return ; } const downgrade = Object . keys ( migrationVersion ) . find ( ( k ) => ! docVersion . hasOwnProperty ( k ) || Semver . lt ( docVersion [ k ] , migrationVersion [ k ] ) ) ; if ( downgrade ) { throw new Error ( `Migration "${ prop } v ${ version }" attempted to ` + `downgrade "migrationVersion.${ downgrade }" from ${ migrationVersion [ downgrade ] } ` + `to ${ docVersion [ downgrade ] }.` ) ; } } note internal export function deterministicallyRegenerateObjectId ( namespace , type , id ) { return uuidv5 ( `${ namespace }:${ type }:${ id }` , uuidv5 . DNS ) ; }