import assert = require ( "assert" ) ; import { ChildProcess , exec as node_exec , fork } from "child_process" ; import * as crypto from "crypto" ; import moment = require ( "moment" ) ; import * as os from "os" ; import { Options } from "../lib/common" ; import ProgressBar from "./progress" ; export function assertDefined < T > ( x , message ? ) { assert ( x !== undefined , message ) ; return x ! ; } const DEFAULT_CRASH_RECOVERY_MAX_OLD_SPACE_SIZE = 4096 ; export function parseJson ( text ) { try { return JSON . parse ( text ) as object ; } catch ( err ) { throw new Error ( `${ ( err as Error ) . message } due to JSON: ${ text }` ) ; } } export function currentTimeStamp ( ) { return moment ( ) . format ( "YYYY-MM-DDTHH:mm:ss.SSSZZ" ) ; } export const numberOfOsProcesses = process . env . TRAVIS === "true" ? 2 : os . cpus ( ) . length ; /** Progress options needed for `nAtATime`. Other options will be inferred. */ interface ProgressOptions < T , U > { readonly name : string ; flavor ( input , output ) : string | undefined ; readonly options : Options ; } export async function nAtATime < T , U > ( n , inputs , use , progressOptions ? ) { const progress = progressOptions && progressOptions . options . progress ? new ProgressBar ( { name : progressOptions . name } ) : undefined ; const results = new Array ( inputs . length ) ; let nextIndex = 0 ; await Promise . all ( initArray ( n , async ( ) => { while ( nextIndex !== inputs . length ) { const index = nextIndex ; nextIndex ++ ; const input = inputs [ index ] ; const output = await use ( input ) ; results [ index ] = output ; if ( progress ) { progress ! . update ( index / inputs . length , progressOptions ! . flavor ( input , output ) ) ; } } } ) ) ; if ( progress ) { progress . done ( ) ; } return results ; } export function filter < T > ( iterable , predicate ) { const iter = iterable [ Symbol . iterator ] ( ) ; return { [ Symbol . iterator ] ( ) { return this ; } , next ( ) { while ( true ) { const res = iter . next ( ) ; if ( res . done || predicate ( res . value ) ) { return res ; } } } , } ; } export type Awaitable < T > = T | Promise < T > ; export async function filterNAtATimeOrdered < T > ( n , inputs , shouldKeep , progress ? ) { const shouldKeeps = await nAtATime ( n , inputs , shouldKeep , progress ) ; return inputs . filter ( ( _ , idx ) => shouldKeeps [ idx ] ) ; } export function unique < T > ( arr ) { return [ ... new Set ( arr ) ] ; } export function logUncaughtErrors ( promise ) { ( typeof promise === "function" ? promise ( ) : promise ) . catch ( error => { console . error ( error ) ; process . exit ( 1 ) ; } ) ; } function initArray < T > ( length , makeElement ) { const arr = new Array ( length ) ; for ( let i = 0 ; i < length ; i ++ ) { arr [ i ] = makeElement ( i ) ; } return arr ; } /** Always use "/" for consistency. (This affects package content hash.) */ export function joinPaths ( ... paths ) { return paths . join ( "/" ) ; } /** Convert a path to use "/" instead of "\\" for consistency. (This affects content hash.) */ export function normalizeSlashes ( path ) { return path . replace ( /\\/g , "/" ) ; } export function hasWindowsSlashes ( path ) { return path . includes ( "\\" ) ; } export function intOfString ( str ) { const n = Number . parseInt ( str , 10 ) ; if ( Number . isNaN ( n ) ) { throw new Error ( `Error in parseInt(${ JSON . stringify ( str ) })` ) ; } return n ; } export function sortObjectKeys < T extends { [ key ] : unknown ; } > ( data ) { const out = { } as T ; for ( const key of Object . keys ( data ) . sort ( ) ) { out [ key as keyof T ] = data [ key as keyof T ] ; } return out ; } /** Run a command and return the error, stdout, and stderr. (Never throws.) */ export function exec ( cmd , cwd ? ) { return new Promise < { error : Error | undefined ; stdout : string ; stderr : string ; } > ( resolve => { const maxBuffer = 1024 * 1024 * 1 ; node_exec ( cmd , { encoding : "utf8" , cwd , maxBuffer } , ( error , stdout , stderr ) => { resolve ( { error : error === null ? undefined : error , stdout : stdout . trim ( ) , stderr : stderr . trim ( ) } ) ; } ) ; } ) ; } /** Run a command and return the stdout, or if there was an error, throw. */ export async function execAndThrowErrors ( cmd , cwd ? ) { const { error , stdout , stderr } = await exec ( cmd , cwd ) ; if ( error ) { throw new Error ( `${ error . stack }\n${ stderr }` ) ; } return stdout + stderr ; } param isBetter export function best < T > ( inputs , isBetter ) { const iter = inputs [ Symbol . iterator ] ( ) ; const first = iter . next ( ) ; if ( first . done ) { return undefined ; } let res = first . value ; while ( true ) { const { value , done } = iter . next ( ) ; if ( done ) { break ; } if ( isBetter ( value , res ) ) { res = value ; } } return res ; } export function computeHash ( content ) { const normalContent = content . replace ( /\r\n?/g , "\n" ) ; const h = crypto . createHash ( "sha256" ) ; h . update ( normalContent , "utf8" ) ; return h . digest ( "hex" ) ; } export function mapValues < K , V1 , V2 > ( map , valueMapper ) { const out = new Map < K , V2 > ( ) ; map . forEach ( ( value , key ) => { out . set ( key , valueMapper ( value ) ) ; } ) ; return out ; } export function mapDefined < T , U > ( arr , mapper ) { const out = [ ] ; for ( const a of arr ) { const res = mapper ( a ) ; if ( res !== undefined ) { out . push ( res ) ; } } return out ; } export async function mapDefinedAsync < T , U > ( arr , mapper ) { const out = [ ] ; for ( const a of arr ) { const res = await mapper ( a ) ; if ( res !== undefined ) { out . push ( res ) ; } } return out ; } export function * mapIter < T , U > ( inputs , mapper ) { for ( const input of inputs ) { yield mapper ( input ) ; } } export function * flatMap < T , U > ( inputs , mapper ) { for ( const input of inputs ) { yield * mapper ( input ) ; } } export function sort < T > ( values , comparer ? ) { return Array . from ( values ) . sort ( comparer ) ; } export function join < T > ( values , joiner = ", " ) { let s = "" ; for ( const v of values ) { s += `${ v }${ joiner }` ; } return s . slice ( 0 , s . length - joiner . length ) ; } export interface RunWithChildProcessesOptions < In > { readonly inputs : ReadonlyArray < In > ; readonly commandLineArgs : string [ ] ; readonly workerFile : string ; readonly nProcesses : number ; handleOutput ( output ) : void ; } export function runWithChildProcesses < In > ( { inputs , commandLineArgs , workerFile , nProcesses , handleOutput } ) { return new Promise ( ( resolve , reject ) => { const nPerProcess = Math . floor ( inputs . length / nProcesses ) ; let processesLeft = nProcesses ; let rejected = false ; const allChildren = [ ] ; for ( let i = 0 ; i < nProcesses ; i ++ ) { const lo = nPerProcess * i ; const hi = i === nProcesses - 1 ? inputs . length : lo + nPerProcess ; let outputsLeft = hi - lo ; if ( outputsLeft === 0 ) { processesLeft -- ; continue ; } const child = fork ( workerFile , commandLineArgs ) ; allChildren . push ( child ) ; child . send ( inputs . slice ( lo , hi ) ) ; child . on ( "message" , outputMessage => { handleOutput ( outputMessage as unknown ) ; assert ( outputsLeft > 0 ) ; outputsLeft -- ; if ( outputsLeft === 0 ) { assert ( processesLeft > 0 ) ; processesLeft -- ; if ( processesLeft === 0 ) { resolve ( ) ; } child . kill ( ) ; } } ) ; child . on ( "disconnect" , ( ) => { if ( outputsLeft !== 0 ) { fail ( ) ; } } ) ; child . on ( "close" , ( ) => { assert ( rejected || outputsLeft === 0 ) ; } ) ; child . on ( "error" , fail ) ; } function fail ( ) { rejected = true ; for ( const child of allChildren ) { child . kill ( ) ; } reject ( new Error ( "Parsing failed." ) ) ; } } ) ; } export const enum CrashRecoveryState { Normal , Retry , RetryWithMoreMemory , Crashed } interface RunWithListeningChildProcessesOptions < In > { readonly inputs : ReadonlyArray < In > ; readonly commandLineArgs : string [ ] ; readonly workerFile : string ; readonly nProcesses : number ; readonly cwd : string ; readonly crashRecovery ? : boolean ; readonly crashRecoveryMaxOldSpaceSize ? : number ; readonly softTimeoutMs ? : number ; handleOutput ( output , processIndex ) : void ; handleStart ? ( input , processIndex ) : void ; handleCrash ? ( input , state , processIndex ) : void ; } export function runWithListeningChildProcesses < In > ( { inputs , commandLineArgs , workerFile , nProcesses , cwd , handleOutput , crashRecovery , crashRecoveryMaxOldSpaceSize = DEFAULT_CRASH_RECOVERY_MAX_OLD_SPACE_SIZE , handleStart , handleCrash , softTimeoutMs = Infinity } ) { return new Promise ( ( resolve , reject ) => { let inputIndex = 0 ; let processesLeft = nProcesses ; let rejected = false ; const runningChildren = new Set < ChildProcess > ( ) ; const maxOldSpaceSize = getMaxOldSpaceSize ( process . execArgv ) || 0 ; const startTime = Date . now ( ) ; for ( let i = 0 ; i < nProcesses ; i ++ ) { if ( inputIndex === inputs . length ) { processesLeft -- ; continue ; } const processIndex = nProcesses > 1 ? i + 1 : undefined ; let child ; let crashRecoveryState = CrashRecoveryState . Normal ; let currentInput ; const onMessage = ( outputMessage ) => { try { const oldCrashRecoveryState = crashRecoveryState ; crashRecoveryState = CrashRecoveryState . Normal ; handleOutput ( outputMessage as { } , processIndex ) ; if ( inputIndex === inputs . length || Date . now ( ) - startTime > softTimeoutMs ) { stopChild ( true ) ; } else { if ( oldCrashRecoveryState !== CrashRecoveryState . Normal ) { console . log ( `${ processIndex }> Restarting...` ) ; restartChild ( nextTask , process . execArgv ) ; } else { nextTask ( ) ; } } } catch ( e ) { onError ( e ) ; } } ; const onClose = ( ) => { if ( rejected || ! runningChildren . has ( child ) ) { return ; } try { if ( crashRecovery ) { switch ( crashRecoveryState ) { case CrashRecoveryState . Normal : crashRecoveryState = CrashRecoveryState . Retry ; break ; case CrashRecoveryState . Retry : crashRecoveryState = maxOldSpaceSize < crashRecoveryMaxOldSpaceSize ? CrashRecoveryState . RetryWithMoreMemory : crashRecoveryState = CrashRecoveryState . Crashed ; break ; default : crashRecoveryState = CrashRecoveryState . Crashed ; } } else { crashRecoveryState = CrashRecoveryState . Crashed ; } if ( handleCrash ) { handleCrash ( currentInput , crashRecoveryState , processIndex ) ; } switch ( crashRecoveryState ) { case CrashRecoveryState . Retry : restartChild ( resumeTask , process . execArgv ) ; break ; case CrashRecoveryState . RetryWithMoreMemory : restartChild ( resumeTask , [ ... getExecArgvWithoutMaxOldSpaceSize ( ) , `--max_old_space_size=${ crashRecoveryMaxOldSpaceSize }` , ] ) ; break ; case CrashRecoveryState . Crashed : crashRecoveryState = CrashRecoveryState . Normal ; if ( inputIndex === inputs . length || Date . now ( ) - startTime > softTimeoutMs ) { stopChild ( true ) ; } else { restartChild ( nextTask , process . execArgv ) ; } break ; default : assert . fail ( `${ processIndex }> Unexpected crashRecoveryState: ${ crashRecoveryState }` ) ; } } catch ( e ) { onError ( e ) ; } } ; const onError = ( err ? ) => { child . removeAllListeners ( ) ; runningChildren . delete ( child ) ; fail ( err ) ; } ; const startChild = ( taskAction , execArgv ) => { try { child = fork ( workerFile , commandLineArgs , { cwd , execArgv } ) ; runningChildren . add ( child ) ; } catch ( e ) { fail ( e ) ; return ; } try { let closed = false ; const thisChild = child ; const onChildClosed = ( ) => { if ( ! closed && child === thisChild ) { closed = true ; onClose ( ) ; } } ; const onChildDisconnectedOrExited = ( ) => { if ( ! closed && thisChild === child ) { const timeout = 1000 ; setTimeout ( onChildClosed , timeout ) ; } } ; child . on ( "message" , onMessage ) ; child . on ( "close" , onChildClosed ) ; child . on ( "disconnect" , onChildDisconnectedOrExited ) ; child . on ( "exit" , onChildDisconnectedOrExited ) ; child . on ( "error" , onError ) ; taskAction ( ) ; } catch ( e ) { onError ( e ) ; } } ; const stopChild = ( done ) => { try { assert ( runningChildren . has ( child ) , `${ processIndex }> Child not running` ) ; if ( done ) { processesLeft -- ; if ( processesLeft === 0 ) { resolve ( ) ; } } runningChildren . delete ( child ) ; child . removeAllListeners ( ) ; child . kill ( ) ; } catch ( e ) { onError ( e ) ; } } ; const restartChild = ( taskAction , execArgv ) => { try { assert ( runningChildren . has ( child ) , `${ processIndex }> Child not running` ) ; console . log ( `${ processIndex }> Restarting...` ) ; stopChild ( false ) ; startChild ( taskAction , execArgv ) ; } catch ( e ) { onError ( e ) ; } } ; const resumeTask = ( ) => { try { assert ( runningChildren . has ( child ) , `${ processIndex }> Child not running` ) ; child . send ( currentInput ) ; } catch ( e ) { onError ( e ) ; } } ; const nextTask = ( ) => { try { assert ( runningChildren . has ( child ) , `${ processIndex }> Child not running` ) ; currentInput = inputs [ inputIndex ] ; inputIndex ++ ; if ( handleStart ) { handleStart ( currentInput , processIndex ) ; } child . send ( currentInput ) ; } catch ( e ) { onError ( e ) ; } } ; startChild ( nextTask , process . execArgv ) ; } function fail ( err ? ) { if ( ! rejected ) { rejected = true ; for ( const child of runningChildren ) { try { child . removeAllListeners ( ) ; child . kill ( ) ; } catch { } } const message = err ? `: ${ err . message }` : "" ; reject ( new Error ( `Something went wrong in ${ runWithListeningChildProcesses . name }${ message }` ) ) ; } } } ) ; } const maxOldSpaceSizeRegExp = /^--max[-_]old[-_]space[-_]size(?:$|=(\d+))/ ; interface MaxOldSpaceSizeArgument { index : number ; size : number ; value : number | undefined ; } function getMaxOldSpaceSizeArg ( argv ) { for ( let index = 0 ; index < argv . length ; index ++ ) { const match = maxOldSpaceSizeRegExp . exec ( argv [ index ] ) ; if ( match ) { const value = match [ 1 ] ? parseInt ( match [ 1 ] , 10 ) : argv [ index + 1 ] ? parseInt ( argv [ index + 1 ] , 10 ) : undefined ; const size = match [ 1 ] ? 1 : 2 ; return { index , size , value } ; } } return undefined ; } function getMaxOldSpaceSize ( argv ) { const arg = getMaxOldSpaceSizeArg ( argv ) ; return arg && arg . value ; } let execArgvWithoutMaxOldSpaceSize ; function getExecArgvWithoutMaxOldSpaceSize ( ) { if ( ! execArgvWithoutMaxOldSpaceSize ) { const execArgv = process . execArgv . slice ( ) ; let maxOldSpaceSizeArg = getMaxOldSpaceSizeArg ( execArgv ) ; while ( maxOldSpaceSizeArg ) { execArgv . splice ( maxOldSpaceSizeArg . index , maxOldSpaceSizeArg . size ) ; maxOldSpaceSizeArg = getMaxOldSpaceSizeArg ( execArgv ) ; } execArgvWithoutMaxOldSpaceSize = execArgv ; } return execArgvWithoutMaxOldSpaceSize ; } export function assertNever ( _ ) { throw new Error ( ) ; } export function recordToMap < T > ( record ) ; export function recordToMap < T , U > ( record , cb ) ; export function recordToMap < T , U > ( record , cb ? ) { const m = new Map < string , T | U > ( ) ; for ( const key of Object . keys ( record ) ) { m . set ( key , cb ? cb ( record [ key ] ) : record [ key ] ) ; } return m ; } export function mapToRecord < T > ( map ) ; export function mapToRecord < T , U > ( map , cb ) ; export function mapToRecord < T , U > ( map , cb ? ) { const o = { } ; map . forEach ( ( value , key ) => { o [ key ] = cb ? cb ( value ) : value ; } ) ; return o ; } export function identity < T > ( t ) { return t ; } export function withoutStart ( s , start ) { return s . startsWith ( start ) ? s . slice ( start . length ) : undefined ; } export function unmangleScopedPackage ( packageName ) { const separator = "__" ; return packageName . includes ( separator ) ? `@${ packageName . replace ( separator , "/" ) }` : undefined ; } /** Returns [values that cb returned undefined for, defined results of cb]. */ export function split < T , U > ( inputs , cb ) { const keep = [ ] ; const splitOut = [ ] ; for ( const input of inputs ) { const res = cb ( input ) ; if ( res === undefined ) { keep . push ( input ) ; } else { splitOut . push ( res ) ; } } return [ keep , splitOut ] ; } export function assertSorted ( a ) ; export function assertSorted < T > ( a , cb ) ; export function assertSorted < T > ( a , cb = ( t ) => t as unknown as string ) { let prev = a [ 0 ] ; for ( let i = 1 ; i < a . length ; i ++ ) { const x = a [ i ] ; assert ( cb ( x ) >= cb ( prev ) , `${ JSON . stringify ( x ) } >= ${ JSON . stringify ( prev ) }` ) ; prev = x ; } return a ; }