import { createHash } from "crypto" ; import * as libqp from "libqp" ; import { git , revParse } from "./git" ; import { GitNotes } from "./git-notes" ; import { IGitGitGadgetOptions } from "./gitgitgadget" ; import { GitHubGlue } from "./github-glue" ; import { IMailMetadata } from "./mail-metadata" ; import { IPatchSeriesMetadata } from "./patch-series-metadata" ; import { IParsedMBox , parseMBox , parseMBoxMessageIDAndReferences } from "./send-mail" ; import { SousChef } from "./sous-chef" ; const stateKey = "git@vger.kernel.org <-> GitGitGadget" ; const replyToThisURL = "https://github.com/gitgitgadget/gitgitgadget/wiki/ReplyToThis" ; export interface IGitMailingListMirrorState { latestRevision ? : string ; } export class MailArchiveGitHelper { public static async get ( gggNotes , mailArchiveGitDir , githubGlue ) { const state = await gggNotes . get < IGitMailingListMirrorState > ( stateKey ) || { } ; return new MailArchiveGitHelper ( gggNotes , mailArchiveGitDir , githubGlue , state ) ; } param key returns public static hashKey ( key ) { const hash = createHash ( "sha1" , { encoding : "utf8" } ) ; hash . update ( `blob ${ Buffer . byteLength ( key ) + 1 }` ) ; hash . update ( `\0${ key }\n` ) ; return hash . digest ( "hex" ) ; } public static mbox2markdown ( mbox ) { let body = mbox . body ; const headers = mbox . headers || [ ] ; for ( const header of headers ) { if ( header . key === "Content-Transfer-Encoding" ) { const value = header . value . toLowerCase ( ) ; if ( value === "base64" ) { body = Buffer . from ( body , "base64" ) . toString ( ) ; } else if ( value === "quoted-printable" ) { const buffer = libqp . decode ( body ) ; body = buffer . toString ( "utf-8" ) ; } } } if ( ! body . length ) { return "" ; } const wrap = "``````````\n" ; return `${ wrap }${ body }${ body . endsWith ( "\n" ) ? "" : "\n" }${ wrap }` ; } protected readonly state : IGitMailingListMirrorState ; protected readonly gggNotes : GitNotes ; protected readonly mailArchiveGitDir : string ; protected readonly githubGlue : GitHubGlue ; protected constructor ( gggNotes , mailArchiveGitDir , githubGlue , state ) { this . gggNotes = gggNotes ; this . mailArchiveGitDir = mailArchiveGitDir ; this . githubGlue = githubGlue ; this . state = state ; } public async processMails ( prFilter ? ) { const keys = new Set < string > ( ) ; ( await git ( [ "ls-tree" , "-r" , `${ this . gggNotes . notesRef }:` ] , { workDir : this . gggNotes . workDir } ) ) . split ( "\n" ) . map ( ( line ) => { keys . add ( line . substr ( 53 ) . replace ( /\//g , "" ) ) ; } ) ; const seen = ( messageID ) : boolean => { return keys . has ( MailArchiveGitHelper . hashKey ( messageID ) ) ; } ; const handleWhatsCooking = async ( mbox ) : Promise < void > => { const options = await this . gggNotes . get < IGitGitGadgetOptions > ( "" ) ; if ( ! options || ! options . openPRs ) { return ; } const branchNameMap = new Map < string , string > ( ) ; for ( const pullRequestURL of Object . keys ( options . openPRs ) ) { if ( prFilter && ! prFilter ( pullRequestURL ) ) { continue ; } const prMeta = await this . gggNotes . get < IPatchSeriesMetadata > ( pullRequestURL ) ; if ( prMeta && prMeta . branchNameInGitsterGit ) { branchNameMap . set ( prMeta . branchNameInGitsterGit , pullRequestURL ) ; } } const sousChef = new SousChef ( mbox ) ; if ( ! sousChef . messageID ) { throw new Error ( `Could not parse Message-ID of ${ mbox }` ) ; } console . log ( `Handling "${ sousChef . subject }"` ) ; const whatsCookingBaseURL = "https://lore.kernel.org/git/" ; for ( const branchName of sousChef . branches . keys ( ) ) { const pullRequestURL = branchNameMap . get ( branchName ) ; if ( pullRequestURL ) { const branchBaseURL = "https://github.com/gitgitgadget/git/commits/" ; const info = sousChef . branches . get ( branchName ) ; const pre = info ?. text . replace ( /&/g , "&amp;" ) . replace ( /</g , "&lt;" ) . replace ( />/g , "&gt;" ) ; let comment ; if ( ! pre || pre . trim ( ) === "" ) { comment = `The branch [\`${ branchName }\`](${ branchBaseURL }${ branchName }) was mentioned in the "${ info ?. sectionName }" section of the [status updates](${ whatsCookingBaseURL }${ sousChef . messageID }) on the Git mailing list.` ; } else { comment = `There was a [status update](${ whatsCookingBaseURL }${ sousChef . messageID }) in the "${ info ?. sectionName }" section about the branch [\`${ branchName }\`](${ branchBaseURL }${ branchName }) on the Git mailing list:\n\n<pre>\n${ pre }\n</pre>` ; } console . log ( `\n${ pullRequestURL }: ${ comment }` ) ; await this . githubGlue . addPRComment ( pullRequestURL , comment ) ; } } } ; const mboxHandler = async ( mbox ) : Promise < void > => { const parsedMbox = parseMBox ( mbox , true ) ; if ( ! parsedMbox . headers ) { throw new Error ( `Could not parse ${ mbox }` ) ; } const parsed = parseMBoxMessageIDAndReferences ( parsedMbox ) ; if ( parsedMbox . subject ?. match ( /^What's cooking in git.git / ) && parsedMbox . from === "Junio C Hamano <gitster@pobox.com>" ) { return handleWhatsCooking ( mbox ) ; } if ( seen ( parsed . messageID ) ) { console . log ( `Already handled: ${ parsed . messageID }` ) ; return ; } let pullRequestURL ; let originalCommit ; let issueCommentId ; for ( const reference of parsed . references . filter ( seen ) ) { const data = await this . gggNotes . get < IMailMetadata > ( reference ) ; if ( data && data . pullRequestURL ) { if ( prFilter && ! prFilter ( data . pullRequestURL ) ) { continue ; } const commit = reference . match ( /^pull/ ) ? undefined : data . originalCommit ; if ( ! pullRequestURL || ( ! originalCommit && commit ) || ( ! issueCommentId && data . issueCommentId ) ) { pullRequestURL = data . pullRequestURL ; issueCommentId = data . issueCommentId ; originalCommit = commit ; } } } if ( ! pullRequestURL ) { return ; } console . log ( `Message-ID ${ parsed . messageID } (length ${ mbox . length }) for PR ${ pullRequestURL }, commit ${ originalCommit }, comment ID: ${ issueCommentId }` ) ; const archiveURL = `https://lore.kernel.org/git/${ parsed . messageID }` ; const header = `[On the Git mailing list](${ archiveURL }), ` + ( parsedMbox . from ? parsedMbox . from . replace ( / *<.*>/ , "" ) : "Somebody" ) + ` wrote ([reply to this](${ replyToThisURL })):\n\n` ; const comment = header + MailArchiveGitHelper . mbox2markdown ( parsedMbox ) ; if ( issueCommentId ) { await this . githubGlue . addPRCommentReply ( pullRequestURL , issueCommentId , comment ) ; } else if ( originalCommit ) { const result = await this . githubGlue . addPRCommitComment ( pullRequestURL , originalCommit , this . gggNotes . workDir , comment ) ; issueCommentId = result . id ; } else { await this . githubGlue . addPRComment ( pullRequestURL , comment ) ; } await this . githubGlue . addPRCc ( pullRequestURL , parsedMbox . from || "" ) ; await this . gggNotes . set ( parsed . messageID , { issueCommentId , messageID : parsed . messageID , originalCommit , pullRequestURL , } as IMailMetadata ) ; keys . add ( MailArchiveGitHelper . hashKey ( parsed . messageID ) ) ; } ; let buffer = "" ; let counter = 0 ; const lineHandler = async ( line ) : Promise < void > => { if ( line . startsWith ( "@@ " ) ) { const match = line . match ( /^@@ -(\d+,)?\d+ \+(\d+,)?(\d+)?/ ) ; if ( match ) { if ( counter ) { console . log ( `Oops: unprocessed buffer ${ buffer }` ) ; } counter = parseInt ( match [ 3 ] , 10 ) ; buffer = "" ; } } else if ( counter && line . match ( /^[ +]/ ) ) { buffer += line . substr ( 1 ) + "\n" ; if ( -- counter ) { return ; } try { await mboxHandler ( buffer ) ; } catch ( reason ) { console . log ( `${ reason }: skipping` ) ; } } } ; if ( ! this . state . latestRevision ) { this . state . latestRevision = "3b38d8d206c64bf3dc873ba8ae9dbd48ed43f612" ; } else if ( this . state . latestRevision === "205655703b0501ef14e0f0dddf8e57bb726fae97" ) { this . state . latestRevision = "26674e9a36ae1871f69197798d30f6d3d2af7a56" ; } else if ( await revParse ( this . state . latestRevision , this . mailArchiveGitDir ) === undefined ) { const publicInboxGitDir = process . env . PUBLIC_INBOX_DIR ; if ( publicInboxGitDir === undefined ) { throw new Error ( `Commit ${ this . state . latestRevision } not found; need PUBLIC_INBOX_DIR` ) ; } let commitDiff = await git ( [ "show" , this . state . latestRevision , "--" ] , { workDir : publicInboxGitDir } ) ; if ( ! commitDiff ) { throw new Error ( `Could not find ${ this . state . latestRevision } in ${ publicInboxGitDir }` ) ; } const match = commitDiff . match ( /\n\+(Message-ID: [^\n]*)/ ) ; if ( ! match ) { throw new Error ( `Could not find Message-ID in ${ commitDiff }` ) ; } let commit = "HEAD" ; for ( ; ; ) { commit = await git ( [ "log" , "-1" , "--format=%H" , `-S${ match [ 1 ] }` , commit , "--" ] , { workDir : this . mailArchiveGitDir } ) ; if ( ! commit ) { throw new Error ( `Could not find ${ this . state . latestRevision }'s equivalent in ${ this . mailArchiveGitDir }` ) ; } commitDiff = await git ( [ "show" , commit , "--" ] , { workDir : this . mailArchiveGitDir } ) ; if ( commitDiff . indexOf ( `\n+${ match [ 1 ] }\n` ) >= 0 ) { break ; } commit += "^" ; } this . state . latestRevision = commit ; } const head = await revParse ( "master" , this . mailArchiveGitDir ) ; if ( this . state . latestRevision === head ) { return false ; } const range = `${ this . state . latestRevision }..${ head }` ; console . log ( `Handling commit range ${ range }` ) ; await git ( [ "log" , "-p" , "-U99999" , "--reverse" , range ] , { lineHandler , workDir : this . mailArchiveGitDir } ) ; this . state . latestRevision = head ; await this . gggNotes . set ( stateKey , this . state , true ) ; return true ; } }