import * as child_process from "child_process" ; import * as util from 'util' ; import U = pxt . Util ; export async function uploadRefs ( id , repoUrl ) { pxt . log ( `uploading refs starting from ${ id } in ${ repoUrl } to ${ pxt . Cloud . apiRoot }` ) ; let gitCatFile ; let gitCatFileBuf = new U . PromiseBuffer < Buffer > ( ) ; let apiLockAsync = new U . PromiseQueue ( ) ; let gitCache = new Cache < GitObject > ( ) ; let lastUsage = 0 ; let repoPath = '' ; startGitCatFile ( ) ; let visited = { } ; let toCheck = [ ] ; await processCommit ( id ) ; await uploadMissingObjects ( undefined , true ) ; await refreshRefs ( id , repoUrl ) ; killGitCatFile ( ) ; process . exit ( 0 ) ; async function processCommit ( id , uploadParents = false ) { if ( visited [ id ] ) return ; visited [ id ] = true ; await uploadMissingObjects ( id ) ; let obj = await getGitObjectAsync ( id ) ; if ( obj . type != "commit" ) throw new Error ( "bad type" ) ; if ( uploadParents && obj . commit . parents ) { for ( let parent of obj . commit . parents ) { await processCommit ( parent ) ; } } await processTreeEntry ( '000' , obj . commit . tree ) ; } async function processTree ( entries ) { for ( let entry of entries ) { await processTreeEntry ( entry . mode , entry . sha ) ; } } async function processTreeEntry ( mode , id ) { if ( visited [ id ] ) return ; visited [ id ] = true ; await uploadMissingObjects ( id ) ; if ( mode . indexOf ( '1' ) != 0 ) { let obj = await getGitObjectAsync ( id ) ; if ( obj . type == 'tree' ) { await processTree ( obj . tree ) ; } else { throw new Error ( "bad entry type: " + obj . type ) ; } } } function maybeGcGitCatFile ( ) { if ( ! gitCatFile ) return ; let d = Date . now ( ) - lastUsage ; if ( d < 3000 ) return ; gitCatFile . stdin . end ( ) ; gitCatFile = null ; gitCatFileBuf . drain ( ) ; } function startGitCatFile ( ) { if ( ! lastUsage ) { setInterval ( maybeGcGitCatFile , 1000 ) ; } lastUsage = Date . now ( ) ; if ( ! gitCatFile ) { gitCatFile = child_process . spawn ( "git" , [ "cat-file" , "--batch" ] , { cwd : repoPath , env : process . env , stdio : "pipe" , shell : false } ) ; gitCatFile . stderr . setEncoding ( "utf8" ) ; gitCatFile . stderr . on ( 'data' , ( msg ) => { console . error ( "[git cat-file error] " + msg ) ; } ) ; gitCatFile . stdout . on ( 'data' , ( buf ) => gitCatFileBuf . push ( buf ) ) ; } } function killGitCatFile ( ) { gitCatFile . kill ( ) ; } async function uploadMissingObjects ( id , force ? ) { if ( id ) toCheck . push ( id ) ; if ( toCheck . length > 50 || force ) { let hashes = toCheck ; toCheck = [ ] ; console . log ( `checking hashes with cloud` ) ; let response = await pxt . Cloud . privateRequestAsync ( { url : 'upload/status' , data : { hashes : hashes } } ) ; let missingHashes = response . json . missing ; for ( let missing of missingHashes ) { let obj = await getGitObjectAsync ( missing ) ; console . log ( `uploading raw ${ missing } with type ${ obj . type } to cloud` ) ; await pxt . Cloud . privateRequestAsync ( { url : `upload/raw` , data : { type : obj . type , content : obj . data . toString ( 'base64' ) , encoding : 'base64' , hash : missing } } ) ; } } } async function refreshRefs ( id , repoUrl ) { console . log ( "Updating refs" ) ; const data = { HEAD : id , repoUrl : repoUrl } ; await pxt . Cloud . privateRequestAsync ( { url : `upload/rawrefs` , data : data } ) ; } function getGitObjectAsync ( id ) { if ( ! id || /[\r\n]/ . test ( id ) ) throw new Error ( "bad id: " + id ) ; let cached = gitCache . get ( id ) ; if ( cached ) return Promise . resolve ( cached ) ; return apiLockAsync . enqueue ( "cat-file" , ( ) => { cached = gitCache . get ( id ) ; if ( cached ) return Promise . resolve ( cached ) ; startGitCatFile ( ) ; gitCatFile . stdin . write ( id + "\n" ) ; let sizeLeft = 0 ; let bufs = [ ] ; let res = { id : id , type : "" , memsize : 64 , data : null } ; let typeBuf = null ; let loop = ( ) : Promise < GitObject > => gitCatFileBuf . shiftAsync ( ) . then ( buf => { startGitCatFile ( ) ; if ( ! res . type ) { if ( typeBuf ) { buf = Buffer . concat ( [ typeBuf , buf ] ) ; typeBuf = null ; } else { while ( buf [ 0 ] == 10 ) buf = buf . slice ( 1 ) ; } let end = buf . indexOf ( 10 ) ; if ( end < 0 ) { if ( buf . length == 0 ) { } else { typeBuf = buf ; } return loop ( ) ; } let line = buf ; if ( end >= 0 ) { line = buf . slice ( 0 , end ) ; buf = buf . slice ( end + 1 ) ; } else { throw new Error ( "bad cat-file respose: " + buf . toString ( "utf8" ) . slice ( 0 , 100 ) ) ; } let lineS = line . toString ( "utf8" ) ; if ( / missing/ . test ( lineS ) ) { throw new Error ( "file missing" ) ; } let m = /^([0-9a-f]{40}) (\S+) (\d+)/ . exec ( lineS ) ; if ( ! m ) throw new Error ( "invalid cat-file response: " + lineS + " <nl> " + buf . toString ( "utf8" ) ) ; res . id = m [ 1 ] ; res . type = m [ 2 ] ; sizeLeft = parseInt ( m [ 3 ] ) ; res . memsize += sizeLeft ; } if ( buf . length > sizeLeft ) { buf = buf . slice ( 0 , sizeLeft ) ; } bufs . push ( buf ) ; sizeLeft -= buf . length ; if ( sizeLeft <= 0 ) { res . data = Buffer . concat ( bufs ) ; return res ; } else { return loop ( ) ; } } ) ; return loop ( ) . then ( obj => { if ( obj . type == "tree" ) { obj . tree = parseTree ( obj . data ) ; } else if ( obj . type == "commit" ) { obj . commit = parseCommit ( obj . data ) ; } if ( /^[0-9a-f]{40}/ . test ( id ) ) { gitCache . set ( id , obj , obj . memsize ) ; } return obj ; } ) ; } ) ; } } export interface GitObject { id : string ; type : string ; memsize : number ; data : Buffer ; tree ? : TreeEntry [ ] ; commit ? : Commit ; } export interface Commit { tree : string ; parents : string [ ] ; author : string ; date : number ; msg : string ; } export interface TreeEntry { mode : string ; name : string ; sha : string ; } export type SMap < T > = { [ s ] : T ; } ; interface QEntry { run : ( ) => Promise < any > ; resolve : ( v : any ) => void ; reject : ( err : any ) => void ; } const maxCacheSize = 32 * 1024 * 1024 ; const maxCacheEltSize = 256 * 1024 ; export class Cache < T > { cache : SMap < T > = { } ; size = 0 ; get ( key ) { if ( ! key ) return null ; if ( this . cache . hasOwnProperty ( key ) ) return this . cache [ key ] ; return null ; } set ( key , v , sz ) { if ( ! key ) return ; delete this . cache [ key ] ; if ( ! v || sz > maxCacheEltSize ) return ; if ( this . size + sz > maxCacheSize ) { this . flush ( ) ; } this . size += sz ; this . cache [ key ] = v ; } flush ( ) { this . size = 0 ; this . cache = { } ; } } export function splitName ( fullname ) { let m = /(.*)\/([^\/]+)/ . exec ( fullname ) ; let parent = null ; let name = "" ; if ( ! m ) { if ( fullname == "/" ) { } else if ( fullname . indexOf ( "/" ) == - 1 ) { parent = "/" ; name = fullname ; } else { throw new Error ( "bad name" ) ; } } else { parent = m [ 1 ] || "/" ; name = m [ 2 ] ; } return { parent , name } ; } function parseTree ( buf ) { let entries = [ ] ; let ptr = 0 ; while ( ptr < buf . length ) { let start = ptr ; while ( 48 <= buf [ ptr ] && buf [ ptr ] <= 55 ) ptr ++ ; if ( buf [ ptr ] != 32 ) throw new Error ( "bad tree format" ) ; let mode = buf . slice ( start , ptr ) . toString ( "utf8" ) ; ptr ++ ; start = ptr ; while ( buf [ ptr ] ) ptr ++ ; if ( buf [ ptr ] != 0 ) throw new Error ( "bad tree format 2" ) ; let name = buf . slice ( start , ptr ) . toString ( "utf8" ) ; ptr ++ ; let sha = buf . slice ( ptr , ptr + 20 ) . toString ( "hex" ) ; ptr += 20 ; if ( ptr > buf . length ) throw new Error ( "bad tree format 3" ) ; entries . push ( { mode , name , sha } ) ; } return entries ; } function parseCommit ( buf ) { let cmt = buf . toString ( "utf8" ) ; let mtree = /^tree (\S+)/m . exec ( cmt ) ; let mpar = /^parent (.+)/m . exec ( cmt ) ; let mauthor = /^author (.+) (\d+) ([+\-]\d{4})$/m . exec ( cmt ) ; let midx = cmt . indexOf ( "\n\n" ) ; return { tree : mtree [ 1 ] , parents : mpar ? mpar [ 1 ] . split ( /\s+/ ) : undefined , author : mauthor [ 1 ] , date : parseInt ( mauthor [ 2 ] ) , msg : cmt . slice ( midx + 2 ) } ; }