import { moduleMetaInfoFileName , TFrontendDependency , TScriptMetaInfo } from '@cromwell/core' ; import { getBundledModulesDir , getLogger } from '@cromwell/core-backend' ; import colorsdef from 'colors/safe' ; import extractZip from 'extract-zip' ; import fs from 'fs-extra' ; import moduleDownloader from 'github-directory-downloader' ; import { glob } from 'glob' ; import fetch from 'node-fetch' ; import normalizePath from 'normalize-path' ; import { dirname , resolve } from 'path' ; import readline from 'readline' ; import { promisify } from 'util' ; import { collectFrontendDependencies , collectPackagesInfo , globPackages , parseFrontendDeps } from './shared' ; import { TPackage } from './types' ; const streamPipeline = promisify ( require ( 'stream' ) . pipeline ) ; const colors = colorsdef ; const logger = getLogger ( ) ; export const downloader = async ( options ? ) => { let { rootDir , packages } = options ?? { } ; const { targetModule } = options ?? { } ; rootDir = rootDir ?? process . cwd ( ) ; let downloads = 0 ; let successfulDownloads = 0 ; let frontendDeps = [ ] ; if ( targetModule ) { frontendDeps = await parseFrontendDeps ( [ targetModule ] ) ; } else { if ( ! packages ) { const packagePaths = await globPackages ( rootDir ) ; packages = collectPackagesInfo ( packagePaths ) ; } if ( ! packages ) { logger . error ( 'No packages found' ) ; return ; } frontendDeps = await collectFrontendDependencies ( packages , true ) ; } frontendDeps = frontendDeps . filter ( dep => { if ( dep . name . startsWith ( '@cromwell' ) ) return false ; return true ; } ) ; const bundledModulesDir = normalizePath ( getBundledModulesDir ( ) ) ; await fs . ensureDir ( bundledModulesDir ) ; const processedDependencies = [ ] ; const downloadDepsRecursively = async ( depName ) => { if ( processedDependencies . includes ( depName ) ) return ; processedDependencies . push ( depName ) ; const depDir = resolve ( bundledModulesDir , depName ) ; if ( ! await fs . pathExists ( depDir ) ) { readline . clearLine ( process . stdout , - 1 ) ; readline . clearLine ( process . stdout , 0 ) ; process . stdout . write ( colors . cyan ( `Cromwell:: Downloading frontend module: ${ colors . brightCyan ( depName ) }` ) ) ; readline . cursorTo ( process . stdout , 0 ) ; downloads ++ ; const success = await downloadBundleZipped ( depName , bundledModulesDir ) ; if ( ! success ) { colors . cyan ( `Cromwell:: Downloading frontend module, second attempt: ${ colors . brightCyan ( depName ) }` ) ; const success2 = await downloadBundleZipped ( depName , bundledModulesDir ) ; if ( ! success2 ) return ; } successfulDownloads ++ ; } let meta ; try { meta = require ( resolve ( depDir , moduleMetaInfoFileName ) ) ; if ( meta ?. externalDependencies ) { const subdeps = Object . keys ( meta . externalDependencies ) ; for ( const subdep of subdeps ) { await downloadDepsRecursively ( subdep ) ; } } } catch ( e ) { } } ; const requests = 10 ; await parallelFlows ( requests , frontendDeps , ( dep ) => { const depName = `${ dep . name }@${ dep . version }` ; return downloadDepsRecursively ( depName ) ; } ) ; const allDeps = glob . sync ( bundledModulesDir + '/**/*.zip' ) . map ( zipPath => { return normalizePath ( dirname ( zipPath ) ) . replace ( bundledModulesDir + '/' , '' ) ; } ) ; for ( const dep of allDeps ) { if ( ! processedDependencies . includes ( dep ) ) { await fs . remove ( resolve ( bundledModulesDir , dep ) ) ; } } if ( downloads > 0 ) { readline . clearLine ( process . stdout , - 1 ) ; readline . clearLine ( process . stdout , 0 ) ; console . log ( '\n' ) ; console . log ( colors . cyan ( `Cromwell:: Downloaded ${ successfulDownloads }/${ downloads } modules` ) ) ; } } ; export const downloadBundle = async ( moduleName , saveTo ) => { const repo = 'https://github.com/CromwellCMS/bundled-modules/tree/master' ; const url = `${ repo }/${ moduleName }` ; const stats = await moduleDownloader ( url , resolve ( saveTo , moduleName ) ) ; if ( ! stats . success || stats . downloaded === 0 ) { logger . error ( `Failed to download module ${ moduleName }` + stats . error ) ; await fs . remove ( resolve ( saveTo , moduleName ) ) ; } } ; export const downloadBundleZipped = async ( moduleName , saveTo ) : Promise < boolean > => { const repo = 'https://raw.githubusercontent.com/CromwellCMS/bundled-modules/master' ; const url = `${ repo }/${ moduleName }/module.zip` ; let responseBody ; try { const response = await fetch ( url ) ; if ( ! response . ok ) { throw new Error ( `HTTP ${ response . statusText } for ${ url }` ) ; } responseBody = response . body ; } catch ( e ) { logger . error ( `Failed to download module ${ moduleName }` + e ) ; return false ; } const moduleDir = resolve ( saveTo , moduleName ) ; const zipPath = resolve ( moduleDir , 'module.zip' ) ; await fs . ensureDir ( moduleDir ) ; await streamPipeline ( responseBody , fs . createWriteStream ( zipPath ) ) ; await new Promise ( done => setTimeout ( done , 500 ) ) ; try { await extractZip ( zipPath , { dir : moduleDir } ) ; } catch ( err ) { logger . error ( `Failed to unzip module ${ moduleName }` + err ) ; await fs . remove ( moduleDir ) ; return false ; } return true ; } ; const parallelFlows = async < T > ( flowQnt , items , worker ) => { const statuses = [ ] ; for ( let i = 0 ; i < items . length ; i ++ ) { const num = i % flowQnt ; if ( statuses [ num ] ) { await statuses [ num ] ; } const item = items [ i ] ; statuses [ num ] = worker ( item ) ; } await Promise . all ( statuses ) ; } ;