import chunk from "lodash/chunk" ; import loadJson from "load-json-file" ; import fs from "fs-extra" ; import path from "path" ; import { File } from "@webiny/api-file-manager/types" ; import sleep from "./sleep" ; import downloadInstallationFiles from "./downloadInstallFiles" ; import { PbContext } from "~/graphql/types" ; const FILES_COUNT_IN_EACH_BATCH = 15 ; export default async ( { context } ) : Promise < Record < string , File > > => { /**
     * This function contains logic of file download from S3.
     * Current we're not mocking zip file download from S3 in tests at the moment.
     * So, we're manually mocking it in case of test just by returning an empty object.
     */ if ( process . env . NODE_ENV === "test" ) { return { } ; } const INSTALL_EXTRACT_DIR = await downloadInstallationFiles ( ) ; const pagesFilesData = await loadJson < Record < string , any > [ ] > ( path . join ( INSTALL_EXTRACT_DIR , "data/pagesFilesData.json" ) ) ; try { const fileIdToFileMap = { } ; const chunksProcesses = [ ] ; const filesChunks = chunk ( pagesFilesData , FILES_COUNT_IN_EACH_BATCH ) ; for ( let i = 0 ; i < filesChunks . length ; i ++ ) { chunksProcesses . push ( new Promise ( async ( promise , reject ) => { try { const filesChunk = filesChunks [ i ] ; const s3UploadProcess = [ ] ; for ( let j = 0 ; j < filesChunk . length ; j ++ ) { const currentFile = filesChunk [ j ] ; fileIdToFileMap [ currentFile . id ] = currentFile . type ; try { const buffer = fs . readFileSync ( path . join ( INSTALL_EXTRACT_DIR , "images/" , currentFile . __physicalFileName ) ) ; s3UploadProcess . push ( context . fileManager . storage . upload ( { buffer , size : buffer . length , name : currentFile . name , type : currentFile . type , keyPrefix : "welcome-to-webiny-page" , hideInFileManager : Boolean ( currentFile . meta . private ) } ) ) ; } catch ( e ) { console . log ( "Error while uploading file: " , currentFile . name ) ; console . log ( e ) ; /**
                             * In case of error he still had a fake key so that we get same number of results as files chunk.
                             */ s3UploadProcess . push ( { key : currentFile . key + "/not-found" } ) ; } } const fileUploadResults = await Promise . all ( s3UploadProcess ) ; fileUploadResults . forEach ( ( item , index ) => { fileIdToFileMap [ filesChunk [ index ] . id ] = item ; } ) ; promise ( fileUploadResults ) ; } catch ( e ) { reject ( e ) ; } } ) ) ; await sleep ( 750 ) ; } await Promise . all ( chunksProcesses ) ; return fileIdToFileMap ; } catch ( e ) { console . log ( `[savePageAssets]: error occurred: ${ e . stack }` ) ; } } ; interface UploadPageAssetsParams { context : PbContext ; fileData : any [ ] ; keyPrefix : string ; assetsDirName : string ; assetDataKey : string ; } interface UploadPageAssetsReturnType { fileIdToKeyMap ? : Record < string , string > ; } export const uploadPageAssets = async ( { context , fileData , keyPrefix , assetsDirName = "images/" , assetDataKey = "__physicalFileName" } ) : Promise < UploadPageAssetsReturnType > => { /**
     * This function contains logic of file download from S3.
     * Current we're not mocking zip file download from S3 in tests at the moment.
     * So, we're manually mocking it in case of test just by returning an empty object.
     */ if ( process . env . NODE_ENV === "test" ) { return { } ; } try { const fileIdToKeyMap = { } ; const chunksProcesses = [ ] ; const filesChunks = chunk ( fileData , FILES_COUNT_IN_EACH_BATCH ) ; for ( let i = 0 ; i < filesChunks . length ; i ++ ) { chunksProcesses . push ( new Promise ( async ( promise , reject ) => { try { const filesChunk = filesChunks [ i ] ; const s3UploadProcess = [ ] ; for ( let j = 0 ; j < filesChunk . length ; j ++ ) { const currentFile = filesChunk [ j ] ; fileIdToKeyMap [ currentFile . id ] = currentFile . type ; try { const buffer = fs . readFileSync ( path . join ( assetsDirName , currentFile [ assetDataKey ] ) ) ; s3UploadProcess . push ( context . fileManager . storage . upload ( { buffer , size : buffer . length , name : removeKeyPrefixFromName ( currentFile . name ) , type : currentFile . type , keyPrefix , hideInFileManager : Boolean ( currentFile . meta && currentFile . meta . private ) } ) ) ; } catch ( e ) { console . log ( "Error while uploading file: " , currentFile . name ) ; console . log ( e ) ; /**
                             * In case of error he still had a fake key so that we get same number of results as files chunk.
                             */ s3UploadProcess . push ( { key : currentFile . key + "/not-found" } ) ; } } const fileUploadResults = await Promise . all ( s3UploadProcess ) ; fileUploadResults . forEach ( ( item , index ) => { fileIdToKeyMap [ filesChunk [ index ] . id ] = item . key ; } ) ; promise ( fileUploadResults ) ; } catch ( e ) { console . log ( "REJECT => " , e ) ; reject ( e ) ; } } ) ) ; await sleep ( 750 ) ; } await Promise . all ( chunksProcesses ) ; return { fileIdToKeyMap } ; } catch ( e ) { console . log ( `[savePageAssets]: error occurred: ${ e . stack }` ) ; } } ; param workingDir param ext return export const getFileNameByExtension = ( workingDir , ext ) : string => { const dirContents = fs . readdirSync ( workingDir ) ; return dirContents . find ( a => a . endsWith ( ext ) ) ; } ; function removeKeyPrefixFromName ( name ) { /**
     * Because we prefix unique ID to file key with "-".
     */ const tokens = name . split ( "-" ) ; if ( tokens . length > 1 ) { return tokens . slice ( 1 ) . join ( "-" ) ; } return name ; }