import * as child_process from 'child_process' ; import * as fs from 'fs' ; import * as zlib from 'zlib' ; import * as url from 'url' ; import * as http from 'http' ; import * as https from 'https' ; import * as events from 'events' ; import * as crypto from 'crypto' ; import * as path from 'path' ; import * as os from 'os' ; import Util = pxt . Util ; export interface SpawnOptions { cmd : string ; args : string [ ] ; cwd ? : string ; shell ? : boolean ; pipe ? : boolean ; input ? : string ; silent ? : boolean ; envOverrides ? : pxt . Map < string > ; allowNonZeroExit ? : boolean ; } export let targetDir = process . cwd ( ) ; export let pxtCoreDir = path . join ( __dirname , ".." ) ; export let cliFinalizers = [ ] ; export function addCliFinalizer ( f ) { cliFinalizers . push ( f ) ; } export function runCliFinalizersAsync ( ) { let fins = cliFinalizers ; cliFinalizers = [ ] ; return pxt . Util . promiseMapAllSeries ( fins , f => f ( ) ) . then ( ( ) => { } ) ; } export function setTargetDir ( dir ) { targetDir = dir ; ( < any > module ) . paths . push ( path . join ( targetDir , "node_modules" ) ) ; } export function readResAsync ( g ) { return new Promise < Buffer > ( ( resolve , reject ) => { let bufs = [ ] ; g . on ( 'data' , ( c ) => { if ( typeof c === "string" ) bufs . push ( Buffer . from ( c , "utf8" ) ) ; else bufs . push ( c ) ; } ) ; g . on ( "error" , ( err ) => reject ( err ) ) ; g . on ( 'end' , ( ) => resolve ( Buffer . concat ( bufs ) ) ) ; } ) ; } export function spawnAsync ( opts ) { opts . pipe = false ; return spawnWithPipeAsync ( opts ) . then ( ( ) => { } ) ; } export function spawnWithPipeAsync ( opts ) { if ( opts . pipe === undefined ) opts . pipe = true ; let info = opts . cmd + " " + opts . args . join ( " " ) ; if ( opts . cwd && opts . cwd != "." ) info = "cd " + opts . cwd + "; " + info ; console . log ( "[run] " + info ) ; return new Promise < Buffer > ( ( resolve , reject ) => { let ch = child_process . spawn ( opts . cmd , opts . args , { cwd : opts . cwd , env : opts . envOverrides ? extendEnv ( process . env , opts . envOverrides ) : process . env , stdio : opts . pipe ? [ opts . input == null ? process . stdin : "pipe" , "pipe" , process . stderr ] : "inherit" , shell : opts . shell || false } as any ) ; let bufs = [ ] ; if ( opts . pipe ) ch . stdout . on ( 'data' , ( buf ) => { bufs . push ( buf ) ; if ( ! opts . silent ) { process . stdout . write ( buf ) ; } } ) ; ch . on ( 'close' , ( code ) => { if ( code != 0 && ! opts . allowNonZeroExit ) reject ( new Error ( "Exit code: " + code + " from " + info ) ) ; resolve ( Buffer . concat ( bufs ) ) ; } ) ; if ( opts . input != null ) ch . stdin . end ( opts . input , "utf8" ) ; } ) ; } function extendEnv ( base , overrides ) { let res = { } ; Object . keys ( base ) . forEach ( key => res [ key ] = base [ key ] ) ; Object . keys ( overrides ) . forEach ( key => res [ key ] = overrides [ key ] ) ; return res ; } export function addCmd ( name ) { return name + ( /^win/ . test ( process . platform ) ? ".cmd" : "" ) ; } export function runNpmAsync ( ... args ) { return runNpmAsyncWithCwd ( "." , ... args ) ; } export interface NpmRegistry { _id : string ; _name : string ; "dist-tags" : pxt . Map < string > ; "versions" : pxt . Map < any > ; } export function npmRegistryAsync ( pkg ) { return Util . httpGetJsonAsync ( `https://registry.npmjs.org/${ pkg }` ) ; } export function runNpmAsyncWithCwd ( cwd , ... args ) { return spawnAsync ( { cmd : addCmd ( "npm" ) , args : args , cwd } ) ; } export function runGitAsync ( ... args ) { return spawnAsync ( { cmd : "git" , args : args , cwd : "." } ) ; } export function gitInfoAsync ( args , cwd ? , silent = false ) { return Promise . resolve ( ) . then ( ( ) => spawnWithPipeAsync ( { cmd : "git" , args : args , cwd , silent } ) ) . then ( buf => buf . toString ( "utf8" ) . trim ( ) ) ; } export function currGitTagAsync ( ) { return gitInfoAsync ( [ "describe" , "--tags" , "--exact-match" ] ) . then ( t => { if ( ! t ) Util . userError ( "no git tag found" ) ; return t ; } ) ; } export function needsGitCleanAsync ( ) { return Promise . resolve ( ) . then ( ( ) => spawnWithPipeAsync ( { cmd : "git" , args : [ "status" , "--porcelain" , "--untracked-files=no" ] } ) ) . then ( buf => { if ( buf . length ) Util . userError ( "Please commit all files to git before running 'pxt bump'" ) ; } ) ; } function nodeHttpRequestAsync ( options ) { let isHttps = false ; let u = < http . RequestOptions > < any > url . parse ( options . url ) ; if ( u . protocol == "https:" ) isHttps = true ; else if ( u . protocol == "http:" ) isHttps = false ; else return Promise . reject ( "bad protocol: " + u . protocol ) ; u . headers = Util . clone ( options . headers ) || { } ; let data = options . data ; u . method = options . method || ( data == null ? "GET" : "POST" ) ; let buf = null ; u . headers [ "accept-encoding" ] = "gzip" ; u . headers [ "user-agent" ] = "PXT-CLI" ; let gzipContent = false ; if ( data != null ) { if ( Buffer . isBuffer ( data ) ) { buf = data ; } else if ( typeof data == "object" ) { buf = Buffer . from ( JSON . stringify ( data ) , "utf8" ) ; u . headers [ "content-type" ] = "application/json; charset=utf8" ; if ( options . allowGzipPost ) gzipContent = true ; } else if ( typeof data == "string" ) { buf = Buffer . from ( data , "utf8" ) ; if ( options . allowGzipPost ) gzipContent = true ; } else { Util . oops ( "bad data" ) ; } } if ( gzipContent ) { buf = zlib . gzipSync ( buf ) ; u . headers [ 'content-encoding' ] = "gzip" ; } if ( buf ) u . headers [ 'content-length' ] = buf . length ; return new Promise < Util . HttpResponse > ( ( resolve , reject ) => { const handleResponse = ( res ) => { let g = res ; if ( /gzip/ . test ( res . headers [ 'content-encoding' ] ) ) { let tmp = zlib . createUnzip ( ) ; res . pipe ( tmp ) ; g = tmp ; } resolve ( readResAsync ( g ) . then ( buf => { let text = null ; try { text = buf . toString ( "utf8" ) ; } catch ( e ) { } let resp = { statusCode : res . statusCode , headers : res . headers , buffer : buf , text : text } ; return resp ; } ) ) ; } ; const req = isHttps ? https . request ( u , handleResponse ) : http . request ( u , handleResponse ) ; req . on ( 'error' , ( err ) => reject ( err ) ) ; req . end ( buf ) ; } ) ; } function sha256 ( hashData ) { let sha ; let hash = crypto . createHash ( "sha256" ) ; hash . update ( hashData , "utf8" ) ; sha = hash . digest ( ) . toString ( "hex" ) . toLowerCase ( ) ; return sha ; } function init ( ) { require ( "promise.prototype.finally" ) . shim ( ) ; process . on ( 'unhandledRejection' , e => { throw e ; } ) ; Util . isNodeJS = true ; Util . httpRequestCoreAsync = nodeHttpRequestAsync ; Util . sha256 = sha256 ; Util . cpuUs = ( ) => { const p = process . cpuUsage ( ) ; return p . system + p . user ; } ; Util . getRandomBuf = buf => { let tmp = crypto . randomBytes ( buf . length ) ; for ( let i = 0 ; i < buf . length ; ++ i ) buf [ i ] = tmp [ i ] ; } ; ( global as any ) . btoa = ( str ) => Buffer . from ( str , "binary" ) . toString ( "base64" ) ; ( global as any ) . atob = ( str ) => Buffer . from ( str , "base64" ) . toString ( "binary" ) ; } export function sanitizePath ( path ) { return path . replace ( /[^\w@\/]/g , "-" ) . replace ( /^\/+/ , "" ) ; } export function readJson ( fn ) { return JSON . parse ( fs . readFileSync ( fn , "utf8" ) ) ; } export function readText ( fn ) { return fs . readFileSync ( fn , "utf8" ) ; } export function readPkgConfig ( dir ) { const fn = path . join ( dir , pxt . CONFIG_NAME ) ; const js = readJson ( fn ) ; const ap = js . additionalFilePath ; if ( ap ) { let adddir = path . join ( dir , ap ) ; if ( ! existsDirSync ( adddir ) ) pxt . U . userError ( `additional pxt.json not found: ${ adddir } in ${ dir } + ${ ap }` ) ; pxt . debug ( "additional pxt.json: " + adddir ) ; const js2 = readPkgConfig ( adddir ) ; for ( let k of Object . keys ( js2 ) ) { if ( ! js . hasOwnProperty ( k ) ) { ( js as any ) [ k ] = ( js2 as any ) [ k ] ; } } js . additionalFilePaths = [ ap ] . concat ( js2 . additionalFilePaths . map ( d => path . join ( ap , d ) ) ) ; } else { js . additionalFilePaths = [ ] ; } return js ; } export function getPxtTarget ( ) { if ( fs . existsSync ( targetDir + "/built/target.json" ) ) { let res = readJson ( targetDir + "/built/target.json" ) ; if ( res . id && res . bundledpkgs ) return res ; } let raw = readJson ( targetDir + "/pxtarget.json" ) ; raw . bundledpkgs = { } ; return raw ; } export function pathToPtr ( path ) { return "ptr-" + sanitizePath ( path . replace ( /^ptr-/ , "" ) ) . replace ( /[^\w@]/g , "-" ) ; } export function mkdirP ( thePath ) { if ( thePath == "." || ! thePath ) return ; if ( ! fs . existsSync ( thePath ) ) { mkdirP ( path . dirname ( thePath ) ) ; fs . mkdirSync ( thePath ) ; } } export function cpR ( src , dst , maxDepth = 8 ) { src = path . resolve ( src ) ; let files = allFiles ( src , maxDepth ) ; let dirs = { } ; for ( let f of files ) { let bn = f . slice ( src . length ) ; let dd = path . join ( dst , bn ) ; let dir = path . dirname ( dd ) ; if ( ! Util . lookup ( dirs , dir ) ) { mkdirP ( dir ) ; dirs [ dir ] = true ; } let buf = fs . readFileSync ( f ) ; fs . writeFileSync ( dd , buf ) ; } } export function cp ( srcFile , destDirectory , destName ? ) { mkdirP ( destDirectory ) ; let dest = path . resolve ( destDirectory , destName || path . basename ( srcFile ) ) ; let buf = fs . readFileSync ( path . resolve ( srcFile ) ) ; fs . writeFileSync ( dest , buf ) ; } export function allFiles ( top , maxDepth = 8 , allowMissing = false , includeDirs = false , ignoredFileMarker = undefined ) { let res = [ ] ; if ( allowMissing && ! existsDirSync ( top ) ) return res ; for ( const p of fs . readdirSync ( top ) ) { if ( p [ 0 ] == "." ) continue ; const inner = path . join ( top , p ) ; const st = fs . statSync ( inner ) ; if ( st . isDirectory ( ) ) { if ( ignoredFileMarker && fs . existsSync ( path . join ( inner , ignoredFileMarker ) ) ) continue ; if ( maxDepth > 1 ) Util . pushRange ( res , allFiles ( inner , maxDepth - 1 ) ) ; if ( includeDirs ) res . push ( inner ) ; } else { res . push ( inner ) ; } } return res ; } export function existsDirSync ( name ) { try { const stats = fs . lstatSync ( name ) ; return stats && stats . isDirectory ( ) ; } catch ( e ) { return false ; } } export function writeFileSync ( p , data , options ? ) { mkdirP ( path . dirname ( p ) ) ; fs . writeFileSync ( p , data , options ) ; if ( pxt . options . debug ) { const stats = fs . statSync ( p ) ; pxt . log ( `  + ${ p } ${ stats . size > 1000000 ? ( stats . size / 1000000 ) . toFixed ( 2 ) + ' m' : stats . size > 1000 ? ( stats . size / 1000 ) . toFixed ( 2 ) + 'k' : stats . size }b` ) ; } } export function openUrl ( startUrl , browser ) { if ( ! /^[a-z0-9A-Z#=\.\-\\\/%:\?_&]+$/ . test ( startUrl ) ) { console . error ( "invalid URL to open: " + startUrl ) ; return ; } let cmds = { darwin : "open" , win32 : "start" , linux : "xdg-open" } ; if ( /^win/ . test ( os . platform ( ) ) && ! /^[a-z0-9]+:\/\//i . test ( startUrl ) ) startUrl = startUrl . replace ( '/' , '\\' ) ; else startUrl = startUrl . replace ( '\\' , '/' ) ; console . log ( `opening ${ startUrl }` ) ; if ( browser ) { child_process . spawn ( getBrowserLocation ( browser ) , [ startUrl ] , { detached : true } ) ; } else { child_process . exec ( `${ cmds [ process . platform ] } ${ startUrl }` ) ; } } function getBrowserLocation ( browser ) { let browserPath ; const normalizedBrowser = browser . toLowerCase ( ) ; if ( normalizedBrowser === "chrome" ) { switch ( os . platform ( ) ) { case "win32" : browserPath = "C:/Program Files (x86)/Google/Chrome/Application/chrome.exe" ; break ; case "darwin" : browserPath = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" ; break ; case "linux" : browserPath = "/opt/google/chrome/chrome" ; break ; default : break ; } } else if ( normalizedBrowser === "firefox" ) { browserPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe" ; switch ( os . platform ( ) ) { case "win32" : browserPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe" ; break ; case "darwin" : browserPath = "/Applications/Firefox.app" ; break ; case "linux" : default : break ; } } else if ( normalizedBrowser === "ie" ) { browserPath = "C:/Program Files/Internet Explorer/iexplore.exe" ; } else if ( normalizedBrowser === "safari" ) { browserPath = "/Applications/Safari.app/Contents/MacOS/Safari" ; } if ( browserPath && fs . existsSync ( browserPath ) ) { return browserPath ; } return browser ; } export function fileExistsSync ( p ) { try { let stats = fs . lstatSync ( p ) ; return stats && stats . isFile ( ) ; } catch ( e ) { return false ; } } export let lastResolveMdDirs = [ ] ; export function resolveMd ( root , pathname , md ? ) { const docs = path . join ( root , "docs" ) ; const tryRead = ( fn ) => { if ( fileExistsSync ( fn + ".md" ) ) return fs . readFileSync ( fn + ".md" , "utf8" ) ; if ( fileExistsSync ( fn + "/index.md" ) ) return fs . readFileSync ( fn + "/index.md" , "utf8" ) ; return null ; } ; const targetMd = md ? md : tryRead ( path . join ( docs , pathname ) ) ; if ( targetMd && ! /^\s*#+\s+@extends/m . test ( targetMd ) ) return targetMd ; const dirs = [ path . join ( root , "/node_modules/pxt-core/common-docs" ) , ... getBundledPackagesDocs ( ) ] ; for ( const d of dirs ) { const template = tryRead ( path . join ( d , pathname ) ) ; if ( template ) return pxt . docs . augmentDocs ( template , targetMd ) ; } return undefined ; } export function getBundledPackagesDocs ( ) { const handledDirectories = { } ; const outputDocFolders = [ ] ; for ( const bundledDir of pxt . appTarget . bundleddirs || [ ] ) { getPackageDocs ( bundledDir , outputDocFolders , handledDirectories ) ; } return outputDocFolders ; /**
     * This needs to produce a topologically sorted array of the docs of `dir` and any required packages,
     * such that any package listed as a dependency / additionalFilePath of another
     * package is added to `folders` before the one that requires it.
     */ function getPackageDocs ( packageDir , folders , resolvedDirs ) { if ( resolvedDirs [ packageDir ] ) return ; resolvedDirs [ packageDir ] = true ; const jsonDir = path . join ( packageDir , "pxt.json" ) ; const pxtjson = fs . existsSync ( jsonDir ) && ( readJson ( jsonDir ) as pxt . PackageConfig ) ; if ( pxtjson ) { /**
             * include the package this extends from first;
             * that may have dependencies that overlap with this one or that will later be
             * overwritten by this one
             **/ if ( pxtjson . additionalFilePath ) { getPackageDocs ( path . join ( packageDir , pxtjson . additionalFilePath ) , folders , resolvedDirs ) ; } if ( pxtjson . dependencies ) { Object . keys ( pxtjson . dependencies ) . forEach ( dep => { const parts = /^file:(.+)$/i . exec ( pxtjson . dependencies [ dep ] ) ; if ( parts ) { getPackageDocs ( path . join ( packageDir , parts [ 1 ] ) , folders , resolvedDirs ) ; } } ) ; } } const docsDir = path . join ( packageDir , "docs" ) ; if ( fs . existsSync ( docsDir ) ) { folders . push ( docsDir ) ; } } } export function lazyDependencies ( ) { const deps = { } ; [ path . join ( "node_modules" , "pxt-core" , "package.json" ) , "package.json" ] . filter ( f => fs . existsSync ( f ) ) . map ( f => readJson ( f ) ) . forEach ( config => config && config . lazyDependencies && Util . jsonMergeFrom ( deps , config . lazyDependencies ) ) ; return deps ; } export function lazyRequire ( name , install = false ) { let r ; try { r = require ( name ) ; } catch ( e ) { pxt . debug ( e ) ; pxt . debug ( ( < any > require . resolve ) . paths ( name ) ) ; r = undefined ; } if ( ! r && install ) pxt . log ( `package "${ name }" failed to load, run "pxt npminstallnative" to install native depencencies` ) ; return r ; } export function stringify ( content ) { if ( process . env [ "PXT_ENV" ] === "production" ) { return JSON . stringify ( content ) ; } return JSON . stringify ( content , null , 4 ) ; } init ( ) ;