import { Readable } from 'stream' ; import { BulkErrorSchema , ImportExceptionListItemSchema , ImportExceptionListItemSchemaDecoded , ImportExceptionListSchemaDecoded , ImportExceptionsListSchema , ImportExceptionsResponseSchema , } from '@kbn/securitysolution-io-ts-list-types' ; import { createPromiseFromStreams } from '@kbn/utils' ; import { SavedObjectsClientContract } from 'kibana/server' ; import { chunk } from 'lodash/fp' ; import { importExceptionLists } from './utils/import/import_exception_lists' ; import { importExceptionListItems } from './utils/import/import_exception_list_items' ; import { getTupleErrorsAndUniqueExceptionLists } from './utils/import/dedupe_incoming_lists' ; import { getTupleErrorsAndUniqueExceptionListItems } from './utils/import/dedupe_incoming_items' ; import { createExceptionsStreamFromNdjson , exceptionsChecksFromArray , } from './utils/import/create_exceptions_stream_logic' ; export interface PromiseFromStreams { lists : Array < ImportExceptionListSchemaDecoded | Error > ; items : Array < ImportExceptionListItemSchemaDecoded | Error > ; } export interface ImportExceptionsOk { id ? : string ; item_id ? : string ; list_id ? : string ; status_code : number ; message ? : string ; } export type ImportResponse = ImportExceptionsOk | BulkErrorSchema ; export type PromiseStream = ImportExceptionsListSchema | ImportExceptionListItemSchema | Error ; export interface ImportDataResponse { success : boolean ; success_count : number ; errors : BulkErrorSchema [ ] ; } interface ImportExceptionListAndItemsOptions { exceptions : PromiseFromStreams ; overwrite : boolean ; savedObjectsClient : SavedObjectsClientContract ; user : string ; } interface ImportExceptionListAndItemsAsStreamOptions { exceptionsToImport : Readable ; maxExceptionsImportSize : number ; overwrite : boolean ; savedObjectsClient : SavedObjectsClientContract ; user : string ; } interface ImportExceptionListAndItemsAsArrayOptions { exceptionsToImport : Array < ImportExceptionsListSchema | ImportExceptionListItemSchema > ; maxExceptionsImportSize : number ; overwrite : boolean ; savedObjectsClient : SavedObjectsClientContract ; user : string ; } export type ExceptionsImport = Array < ImportExceptionListItemSchema | ImportExceptionsListSchema > ; export const CHUNK_PARSED_OBJECT_SIZE = 100 ; params params params params params return { ImportExceptionsResponseSchema } export const importExceptionsAsStream = async ( { exceptionsToImport , maxExceptionsImportSize , overwrite , savedObjectsClient , user , } ) : Promise < ImportExceptionsResponseSchema > => { const readStream = createExceptionsStreamFromNdjson ( maxExceptionsImportSize ) ; const [ parsedObjects ] = await createPromiseFromStreams < PromiseFromStreams [ ] > ( [ exceptionsToImport , ... readStream , ] ) ; return importExceptions ( { exceptions : parsedObjects , overwrite , savedObjectsClient , user , } ) ; } ; params params params params params return { ImportExceptionsResponseSchema } export const importExceptionsAsArray = async ( { exceptionsToImport , maxExceptionsImportSize , overwrite , savedObjectsClient , user , } ) : Promise < ImportExceptionsResponseSchema > => { const objectsToImport = exceptionsChecksFromArray ( exceptionsToImport , maxExceptionsImportSize ) ; return importExceptions ( { exceptions : objectsToImport , overwrite , savedObjectsClient , user , } ) ; } ; const importExceptions = async ( { exceptions , overwrite , savedObjectsClient , user , } ) : Promise < ImportExceptionsResponseSchema > => { const [ exceptionListDuplicateErrors , uniqueExceptionLists ] = getTupleErrorsAndUniqueExceptionLists ( exceptions . lists ) ; const [ exceptionListItemsDuplicateErrors , uniqueExceptionListItems ] = getTupleErrorsAndUniqueExceptionListItems ( exceptions . items ) ; const chunkParsedListObjects = chunk ( CHUNK_PARSED_OBJECT_SIZE , uniqueExceptionLists ) ; const chunkParsedItemsObjects = chunk ( CHUNK_PARSED_OBJECT_SIZE , uniqueExceptionListItems ) ; const importExceptionListsResponse = await importExceptionLists ( { isOverwrite : overwrite , listsChunks : chunkParsedListObjects , savedObjectsClient , user , } ) ; const importExceptionListItemsResponse = await importExceptionListItems ( { isOverwrite : overwrite , itemsChunks : chunkParsedItemsObjects , savedObjectsClient , user , } ) ; const importsSummary = { errors : [ ... importExceptionListsResponse . errors , ... exceptionListDuplicateErrors , ... importExceptionListItemsResponse . errors , ... exceptionListItemsDuplicateErrors , ] , success_count_exception_list_items : importExceptionListItemsResponse . success_count , success_count_exception_lists : importExceptionListsResponse . success_count , success_exception_list_items : importExceptionListItemsResponse . errors . length === 0 && exceptionListItemsDuplicateErrors . length === 0 , success_exception_lists : importExceptionListsResponse . errors . length === 0 && exceptionListDuplicateErrors . length === 0 , } ; return { ... importsSummary , success : importsSummary . success_exception_list_items && importsSummary . success_exception_lists , success_count : importsSummary . success_count_exception_lists + importsSummary . success_count_exception_list_items , } ; } ;