import assert = require ( "assert" ) ; import { existsSync , readFileSync } from "fs" ; import { pathExists , remove } from "fs-extra" ; import os = require ( "os" ) ; import * as fold from "travis-fold" ; import * as yargs from "yargs" ; import { FS , getDefinitelyTyped } from "../get-definitely-typed" ; import { Options , TesterOptions } from "../lib/common" ; import { parseVersionFromDirectoryName } from "../lib/definition-parser" ; import { NpmInfo , UncachedNpmInfoClient } from "../lib/npm-client" ; import { AllPackages , DependencyVersion , formatDependencyVersion , NotNeededPackage , PackageId , TypingsData } from "../lib/packages" ; import { sourceBranch , typesDirectoryName } from "../lib/settings" ; import { Semver } from "../lib/versions" ; import { npmInstallFlags } from "../util/io" ; import { consoleLogger , Logger , LoggerWithErrors , loggerWithErrors } from "../util/logging" ; import { assertDefined , CrashRecoveryState , exec , execAndThrowErrors , flatMap , joinPaths , logUncaughtErrors , mapIter , numberOfOsProcesses , runWithListeningChildProcesses , } from "../util/util" ; import { allDependencies , getAffectedPackages } from "./get-affected-packages" ; const perfDir = joinPaths ( os . homedir ( ) , ".dts" , "perf" ) ; const suggestionsDir = joinPaths ( os . homedir ( ) , ".dts" , "suggestions" ) ; if ( ! module . parent ) { if ( yargs . argv . affected ) { logUncaughtErrors ( testAffectedOnly ( Options . defaults ) ) ; } else { const selection = yargs . argv . all ? "all" : yargs . argv . _ [ 0 ] ? new RegExp ( yargs . argv . _ [ 0 ] ) : "affected" ; const options = testerOptions ( ! ! yargs . argv . runFromDefinitelyTyped ) ; logUncaughtErrors ( getDefinitelyTyped ( options , loggerWithErrors ( ) [ 0 ] ) . then ( dt => runTests ( dt , options . definitelyTypedPath , parseNProcesses ( ) , selection ) ) ) ; } } export interface GitDiff { status : "A" | "D" | "M" ; file : string ; } async function testAffectedOnly ( options ) { const changes = getAffectedPackages ( await AllPackages . read ( await getDefinitelyTyped ( options , loggerWithErrors ( ) [ 0 ] ) ) , gitChanges ( await gitDiff ( consoleLogger . info , options . definitelyTypedPath ) ) ) ; console . log ( { changedPackages : changes . changedPackages . map ( t => t . desc ) , dependersLength : changes . dependentPackages . map ( t => t . desc ) . length } ) ; } export function parseNProcesses ( ) { const str = yargs . argv . nProcesses as string | undefined ; if ( ! str ) { return numberOfOsProcesses ; } const nProcesses = Number . parseInt ( str , 10 ) ; if ( Number . isNaN ( nProcesses ) ) { throw new Error ( "Expected nProcesses to be a number." ) ; } return nProcesses ; } export function testerOptions ( runFromDefinitelyTyped ) { return runFromDefinitelyTyped ? { definitelyTypedPath : process . cwd ( ) , progress : false , parseInParallel : true } : Options . defaults ; } export default async function runTests ( dt , definitelyTypedPath , nProcesses , selection ) { const { changedPackages , dependentPackages , allPackages } = await getAffectedPackagesFromDiff ( dt , definitelyTypedPath , selection ) ; console . log ( `Running with ${ nProcesses } processes.` ) ; const typesPath = `${ definitelyTypedPath }/types` ; await doInstalls ( allPackages , [ ... changedPackages , ... dependentPackages ] , typesPath ) ; console . log ( "Testing..." ) ; await doRunTests ( [ ... changedPackages , ... dependentPackages ] , new Set ( changedPackages ) , typesPath , nProcesses ) ; } export async function getAffectedPackagesFromDiff ( dt , definitelyTypedPath , selection ) { const allPackages = await AllPackages . read ( dt ) ; const diffs = await gitDiff ( consoleLogger . info , definitelyTypedPath ) ; if ( diffs . find ( d => d . file === "notNeededPackages.json" ) ) { const uncached = new UncachedNpmInfoClient ( ) ; for ( const deleted of getNotNeededPackages ( allPackages , diffs ) ) { const source = await uncached . fetchNpmInfo ( deleted . libraryName ) ; const typings = await uncached . fetchNpmInfo ( deleted . fullNpmName ) ; checkNotNeededPackage ( deleted , source , typings ) ; } } const affected = selection === "all" ? { changedPackages : allPackages . allTypings ( ) , dependentPackages : [ ] , allPackages } : selection === "affected" ? getAffectedPackages ( allPackages , gitChanges ( diffs ) ) : { changedPackages : allPackages . allTypings ( ) . filter ( t => selection . test ( t . name ) ) , dependentPackages : [ ] , allPackages } ; console . log ( `Testing ${ affected . changedPackages . length } changed packages: ${ affected . changedPackages . map ( t => t . desc ) . toString ( ) }` ) ; console . log ( `Testing ${ affected . dependentPackages . length } dependent packages: ${ affected . dependentPackages . map ( t => t . desc ) . toString ( ) }` ) ; return affected ; } /**
 * 1. find all the deleted files and group by toplevel
 * 2. Make sure that there are no packages left with deleted entries
 * 3. make sure that each toplevel deleted has a matching entry in notNeededPackages
 */ export function getNotNeededPackages ( allPackages , diffs ) { const deletedPackages = new Set ( diffs . filter ( d => d . status === "D" ) . map ( d => assertDefined ( getDependencyFromFile ( d . file ) , `Unexpected file deleted: ${ d . file }
When removing packages, you should only delete files that are a part of removed packages.` ) . name ) ) ; return mapIter ( deletedPackages , p => { if ( allPackages . hasTypingFor ( { name : p , version : "*" } ) ) { throw new Error ( `Please delete all files in ${ p } when adding it to notNeededPackages.json.` ) ; } return assertDefined ( allPackages . getNotNeededPackage ( p ) , `Deleted package ${ p } is not in notNeededPackages.json.` ) ; } ) ; } /**
 * 1. libraryName must exist on npm (SKIPPED and preferably/optionally have been the libraryName in just-deleted header)
 * (SKIPPED 2.) sourceRepoURL must exist and be the npm homepage
 * 3. asOfVersion must be newer than `@types/name@latest` on npm
 * 4. `name@asOfVersion` must exist on npm
 *
 * I skipped (2) because the cached npm info doesn't include it. I might add it later.
 */ export function checkNotNeededPackage ( unneeded , source , typings ) { source = assertDefined ( source , `The entry for ${ unneeded . fullNpmName } in notNeededPackages.json has
"libraryName": "${ unneeded . libraryName }", but there is no npm package with this name.
Unneeded packages have to be replaced with a package on npm.` ) ; typings = assertDefined ( typings , `Unexpected error: @types package not found for ${ unneeded . fullNpmName }` ) ; const latestTypings = Semver . parse ( assertDefined ( typings . distTags . get ( "latest" ) , `Unexpected error: ${ unneeded . fullNpmName } is missing the "latest" tag.` ) ) ; assert ( unneeded . version . greaterThan ( latestTypings ) , `The specified version ${ unneeded . version . versionString } of ${ unneeded . libraryName } must be newer than the version
it is supposed to replace, ${ latestTypings . versionString } of ${ unneeded . fullNpmName }.` ) ; assert ( source . versions . has ( unneeded . version . versionString ) , `The specified version ${ unneeded . version . versionString } of ${ unneeded . libraryName } is not on npm.` ) ; } async function doInstalls ( allPackages , packages , typesPath ) { console . log ( "Installing NPM dependencies..." ) ; for ( const pkg of allDependencies ( allPackages , packages ) ) { const cwd = directoryPath ( typesPath , pkg ) ; if ( ! await pathExists ( joinPaths ( cwd , "package.json" ) ) ) { continue ; } const cmd = `npm install ${ npmInstallFlags }` ; console . log ( `  ${ cwd }: ${ cmd }` ) ; const stdout = await execAndThrowErrors ( cmd , cwd ) ; if ( stdout ) { console . log ( ` from ${ cwd }: ${ stdout }` ) ; } } await runCommand ( console , undefined , require . resolve ( "dtslint" ) , [ "--installAll" ] ) ; } function directoryPath ( typesPath , pkg ) { return joinPaths ( typesPath , pkg . subDirectoryPath ) ; } async function doRunTests ( packages , changed , typesPath , nProcesses ) { await remove ( suggestionsDir ) ; const allFailures = [ ] ; if ( fold . isTravis ( ) ) { console . log ( fold . start ( "tests" ) ) ; } await runWithListeningChildProcesses ( { inputs : packages . map ( p => ( { path : p . subDirectoryPath , onlyTestTsNext : ! changed . has ( p ) , expectOnly : ! changed . has ( p ) } ) ) , commandLineArgs : [ "--listen" ] , workerFile : require . resolve ( "dtslint" ) , nProcesses , crashRecovery : true , crashRecoveryMaxOldSpaceSize : 0 , cwd : typesPath , handleStart ( input , processIndex ) { const prefix = processIndex === undefined ? "" : `${ processIndex }> ` ; console . log ( `${ prefix }${ input . path } START` ) ; } , handleOutput ( output , processIndex ) { const prefix = processIndex === undefined ? "" : `${ processIndex }> ` ; const { path , status } = output as { path : string ; status : string ; } ; if ( status === "OK" ) { console . log ( `${ prefix }${ path } OK` ) ; } else { console . error ( `${ prefix }${ path } failing:` ) ; console . error ( prefix ? status . split ( /\r?\n/ ) . map ( line => `${ prefix }${ line }` ) . join ( "\n" ) : status ) ; allFailures . push ( [ path , status ] ) ; } } , handleCrash ( input , state , processIndex ) { const prefix = processIndex === undefined ? "" : `${ processIndex }> ` ; switch ( state ) { case CrashRecoveryState . Retry : console . warn ( `${ prefix }${ input . path } Out of memory: retrying` ) ; break ; case CrashRecoveryState . RetryWithMoreMemory : console . warn ( `${ prefix }${ input . path } Out of memory: retrying with increased memory (4096M)` ) ; break ; case CrashRecoveryState . Crashed : console . error ( `${ prefix }${ input . path } Out of memory: failed` ) ; allFailures . push ( [ input . path , "Out of memory" ] ) ; break ; default : } } , } ) ; if ( fold . isTravis ( ) ) { console . log ( fold . end ( "tests" ) ) ; } console . log ( "\n\n=== SUGGESTIONS ===\n" ) ; const suggestionLines = [ ] ; for ( const change of changed ) { const pkgPath = change . versionDirectoryName ? change . name + change . versionDirectoryName : change . name ; const path = joinPaths ( suggestionsDir , pkgPath + ".txt" ) ; if ( existsSync ( path ) ) { const suggestions = readFileSync ( path , "utf8" ) . split ( "\n" ) ; suggestionLines . push ( `"${ change . subDirectoryPath }": [${ suggestions . join ( "," ) }]` ) ; } } console . log ( `{${ suggestionLines . join ( "," ) }}` ) ; console . log ( "\n\n=== PERFORMANCE ===\n" ) ; console . log ( "{" ) ; for ( const change of changed ) { const path = joinPaths ( perfDir , change . name + ".json" ) ; if ( existsSync ( path ) ) { const perf = JSON . parse ( readFileSync ( path , "utf8" ) ) as { [ name ] : { typeCount : number ; } ; } ; console . log ( `  "${ change . name }": ${ perf [ change . name ] . typeCount },` ) ; } } console . log ( "}" ) ; if ( allFailures . length === 0 ) { return ; } console . error ( "\n\n=== ERRORS ===\n" ) ; for ( const [ path , error ] of allFailures ) { console . error ( `\n\nError in ${ path }` ) ; console . error ( error ) ; } throw new Error ( `The following packages had errors: ${ allFailures . map ( e => e [ 0 ] ) . join ( ", " ) }` ) ; } interface TesterError { message : string ; } async function runCommand ( log , cwd , cmd , args ) { const nodeCmd = `node ${ cmd } ${ args . join ( " " ) }` ; log . info ( `Running: ${ nodeCmd }` ) ; try { const { error , stdout , stderr } = await exec ( nodeCmd , cwd ) ; if ( stdout ) { log . info ( stdout ) ; } if ( stderr ) { log . error ( stderr ) ; } return error && { message : `${ error . message }\n${ stdout }\n${ stderr }` } ; } catch ( e ) { return e as TesterError ; } } /** Returns all immediate subdirectories of the root directory that have changed. */ export function gitChanges ( diffs ) { const changedPackages = new Map < string , Map < string , DependencyVersion > > ( ) ; for ( const diff of diffs ) { const dep = getDependencyFromFile ( diff . file ) ; if ( dep ) { const versions = changedPackages . get ( dep . name ) ; if ( ! versions ) { changedPackages . set ( dep . name , new Map ( [ [ formatDependencyVersion ( dep . version ) , dep . version ] ] ) ) ; } else { versions . set ( formatDependencyVersion ( dep . version ) , dep . version ) ; } } } return Array . from ( flatMap ( changedPackages , ( [ name , versions ] ) => mapIter ( versions , ( [ _ , version ] ) => ( { name , version } ) ) ) ) ; } export async function gitDiff ( log , definitelyTypedPath ) { try { await run ( `git rev-parse --verify ${ sourceBranch }` ) ; } catch ( _ ) { await run ( `git fetch origin ${ sourceBranch }` ) ; await run ( `git branch ${ sourceBranch } FETCH_HEAD` ) ; } let diff = ( await run ( `git diff ${ sourceBranch } --name-status` ) ) . trim ( ) ; if ( diff === "" ) { diff = ( await run ( `git diff ${ sourceBranch }~1 --name-status` ) ) . trim ( ) ; } return diff . split ( "\n" ) . map ( line => { const [ status , file ] = line . split ( /\s+/ , 2 ) ; return { status : status . trim ( ) , file : file . trim ( ) } as GitDiff ; } ) ; async function run ( cmd ) { log ( `Running: ${ cmd }` ) ; const stdout = await execAndThrowErrors ( cmd , definitelyTypedPath ) ; log ( stdout ) ; return stdout ; } } /**
 * For "types/a/b/c", returns { name: "a", version: "*" }.
 * For "types/a/v3/c", returns { name: "a", version: 3 }.
 * For "x", returns undefined.
 */ function getDependencyFromFile ( file ) { const parts = file . split ( "/" ) ; if ( parts . length <= 2 ) { return undefined ; } const [ typesDirName , name , subDirName ] = parts ; if ( typesDirName !== typesDirectoryName ) { return undefined ; } if ( subDirName ) { const version = parseVersionFromDirectoryName ( subDirName ) ; if ( version !== undefined ) { return { name , version } ; } } return { name , version : "*" } ; }