import { exec } from 'child_process' ; import crypto from 'crypto' ; import os from 'os' ; import path from 'path' ; import { promisify } from 'util' ; import fs from 'fs-extra' ; import logdown from 'logdown' ; import pkginfo from 'npm-registry-package-info' ; const execAsync = promisify ( exec ) ; interface CrawlerResult { data : { [ id ] : { homepage ? : string ; license ? : string ; name : string ; repository ? : { url : string ; } ; } ; } ; } interface License { license : string ; link ? : string ; package : string ; parent ? : string ; platform : string ; } interface Repository { dir : string ; name : string ; url : string ; } const defaultOptions = { devDependencies : true , filter : [ ] , repositories : [ ] , } ; export interface CollectorOptions { /** Add devDependencies */ devDependencies ? : boolean ; /** Which dependencies to filter */ filter ? : string [ ] ; /** Which repositories to check */ repositories : string [ ] ; } export class LicenseCollector { private readonly options : Required < CollectorOptions > ; private readonly logger : logdown . Logger ; private readonly repositories : Repository [ ] ; private readonly dependencies : string [ ] ; private TMP_DIR : string ; constructor ( options ) { this . options = { ... defaultOptions , ... options } ; this . logger = logdown ( '@wireapp/license-collector/LicenseCollector' , { markdown : false , } ) ; this . logger . state . isEnabled = true ; if ( ! this . options . repositories . length ) { throw new Error ( 'No repositories specified' ) ; } this . dependencies = [ ] ; this . repositories = this . options . repositories . map ( url => ( { dir : '' , name : '' , url } ) ) ; this . TMP_DIR = '' ; } private async checkPrerequisites ( ) { const { stderr : stderrVersion } = await execAsync ( 'git --version' ) ; if ( stderrVersion ) { throw new Error ( `No git installation found: ${ stderrVersion }` ) ; } } private async clone ( ) { const gitUrlRegex = new RegExp ( '/(?<name>(.+?))(?:\\.git)?/?$' , 'i' ) ; for ( const index in this . repositories ) { const { url } = this . repositories [ index ] ; const id = crypto . randomBytes ( 10 ) . toString ( 'hex' ) ; const cloneDir = path . join ( this . TMP_DIR , id ) ; const name = gitUrlRegex . exec ( url ) || [ '' , url ] ; this . repositories [ index ] . name = name [ 1 ] ; this . repositories [ index ] . dir = cloneDir ; this . logger . info ( `${ name [ 1 ] }: Cloning "${ url }" into "${ cloneDir }" ...` ) ; const { stderr : stderrClone } = await execAsync ( `git clone --depth 1 "${ url }" "${ cloneDir }"` ) ; if ( stderrClone . includes ( 'fatal' ) ) { throw new Error ( stderrClone ) ; } } } private async createTempDir ( ) { this . TMP_DIR = await fs . mkdtemp ( path . join ( os . tmpdir ( ) , 'license-collector-' ) ) ; } private async findPackageJson ( dir ) { let candidates = [ ] ; dir = path . resolve ( dir ) ; const fileNames = await fs . readdir ( dir ) ; const files = fileNames . filter ( file => ! [ 'node_modules' , '.git' ] . includes ( file ) ) . map ( file => path . join ( dir , file ) ) ; for ( const file of files ) { const lstat = await fs . lstat ( file ) ; if ( lstat . isDirectory ( ) ) { const moreCandidates = await this . findPackageJson ( file ) ; candidates = candidates . concat ( moreCandidates ) ; } else if ( file . endsWith ( 'package.json' ) ) { candidates . push ( file ) ; } } return candidates ; } private async findRepositories ( ) { for ( const index in this . repositories ) { const { dir : cloneDir , name } = this . repositories [ index ] ; this . logger . info ( `${ name }: Discovering "package.json" files ...` ) ; const packageFiles = await this . findPackageJson ( cloneDir ) ; const packageFileNames = packageFiles . map ( fileName => fileName . replace ( new RegExp ( cloneDir , 'gm' ) , '' ) ) ; this . logger . info ( `${ name }: Found "${ packageFileNames . join ( '", "' ) }"` ) ; this . logger . info ( `${ name }: Discovering direct dependencies ...` ) ; for ( const index in packageFiles ) { let packageJson ; try { packageJson = await fs . readJSON ( packageFiles [ index ] ) ; } catch ( error ) { } const dependencies = Object . keys ( packageJson . dependencies || [ ] ) . filter ( Boolean ) ; const devDependencies = Object . keys ( packageJson . devDependencies || [ ] ) . filter ( Boolean ) ; const plural = ( length ) => ( length === 1 ? 'y' : 'ies' ) ; this . logger . info ( `${ name }: Found ${ dependencies . length } production dependenc${ plural ( dependencies . length ) } and ${ devDependencies . length } dev dependenc${ plural ( devDependencies . length ) } in "${ packageFileNames [ index ] }".` ) ; for ( const dependency of dependencies ) { if ( ! this . dependencies . includes ( dependency ) && ! this . options . filter . includes ( dependency ) ) { this . dependencies . push ( dependency ) ; } } if ( this . options . devDependencies ) { for ( const devDependency of devDependencies ) { if ( ! this . dependencies . includes ( devDependency ) && ! this . options . filter . includes ( devDependency ) ) { this . dependencies . push ( devDependency ) ; } } } } } } private format ( result ) { const licenses = [ ] ; const packages = Object . keys ( result . data ) . sort ( ) ; this . logger . info ( `Extracted ${ packages . length } licenses.` ) ; for ( const packageName of packages ) { const currentPackage = result . data [ packageName ] ; const link = currentPackage . homepage || ( currentPackage . repository ? currentPackage . repository . url : 'none' ) ; const license = { license : currentPackage . license || 'none' , link , package : packageName , platform : 'web/desktop' , } ; licenses . push ( license ) ; } this . logger . info ( `Kept ${ licenses . length } licenses after merging.` ) ; return licenses ; } private async crawl ( ) { this . logger . info ( 'Extracting all licenses ...' ) ; const opts = { packages : this . dependencies , } ; return new Promise ( ( resolve , reject ) => { return pkginfo ( opts , ( error , data ) => { return error ? reject ( error ) : resolve ( data ) ; } ) ; } ) ; } public async collect ( ) { await this . checkPrerequisites ( ) ; await this . createTempDir ( ) ; await this . clone ( ) ; await this . findRepositories ( ) ; const result = await this . crawl ( ) ; await fs . remove ( this . TMP_DIR ) ; return this . format ( result ) ; } }