import { Probot , Context } from 'probot' ; import { PullRequest } from '@octokit/webhooks-definitions/schema' ; import { RequestError } from '@octokit/types' ; import { Configuration , ConfigurationOptions } from './configuration' ; import { DEFAULT_CONFIGURATION , CONFIGURATION_FILE_PATH } from './configuration' ; import { REFRESH_LABEL , NO_PREFIX_REQ_LABEL , SNIPPET_BOT_LABELS } from './labels' ; import { parseRegionTags , parseRegionTagsInPullRequest , ParseResult , } from './region-tag-parser' ; import { Conclusion , CheckAggregator , formatBody , formatExpandable , formatRegionTag , formatViolations , formatMatchingViolation , isFile , } from './utils' ; import { invalidateCache } from './snippets' ; import { Violation , checkProductPrefixViolations , checkRemovingUsedTagViolations , } from './violations' ; import schema from './config-schema.json' ; import { ConfigChecker , getConfig } from '@google-automations/bot-config-utils' ; import { syncLabels } from '@google-automations/label-utils' ; import { logger , addOrUpdateIssueComment } from 'gcf-utils' ; import fetch from 'node-fetch' ; import tmp from 'tmp-promise' ; import tar from 'tar' ; import util from 'util' ; import fs from 'fs' ; import { promises as pfs } from 'fs' ; import path from 'path' ; const streamPipeline = util . promisify ( require ( 'stream' ) . pipeline ) ; interface Label { name : string ; } const FULL_SCAN_ISSUE_TITLE = 'snippet-bot full scan' ; const REFRESH_UI = '- [ ] Refresh this comment' ; const REFRESH_STRING = '- [x] Refresh this comment' ; const MAX_CHARS_IN_COMMENT = 64000 ; async function downloadFile ( url , file ) { const response = await fetch ( url ) ; if ( response . ok ) { return streamPipeline ( response . body , fs . createWriteStream ( file ) ) ; } throw new Error ( `unexpected response ${ response . statusText }` ) ; } async function getFiles ( dir , allFiles ) { const files = ( await pfs . readdir ( dir ) ) . map ( f => path . join ( dir , f ) ) ; for ( const f of files ) { if ( ! ( await pfs . stat ( f ) ) . isDirectory ( ) ) { allFiles . push ( f ) ; } } await Promise . all ( files . map ( async ( f ) => ( await pfs . stat ( f ) ) . isDirectory ( ) && getFiles ( f , allFiles ) ) ) ; return allFiles ; } async function fullScan ( context , configuration ) { const installationId = context . payload . installation ?. id ; const commentMark = `<!-- probot comment [${ installationId }]-->` ; const owner = context . payload . repository . owner . login ; const repo = context . payload . repository . name ; const defaultBranch = context . payload . repository . default_branch ; if ( ! context . payload . issue ?. title . includes ( FULL_SCAN_ISSUE_TITLE ) ) { return ; } const issueNumber = context . payload . issue . number ; const url = `https://github.com/${ owner }/${ repo }/tarball/${ defaultBranch }` ; const tmpDir = tmp . dirSync ( ) ; logger . info ( `working directory: ${ tmpDir . name }` ) ; const file = `${ tmpDir . name }/${ repo }.tar.gz` ; try { await downloadFile ( url , file ) ; logger . info ( `Downloaded to ${ file }` ) ; tar . x ( { file : file , cwd : tmpDir . name , sync : true , } ) ; let archiveDir ! ; for ( const f of await pfs . readdir ( tmpDir . name ) ) { const cur = tmpDir . name + '/' + f ; const stat = await pfs . lstat ( cur ) ; if ( stat . isDirectory ( ) ) { archiveDir = cur ; } } if ( archiveDir === undefined ) { throw new Error ( 'Failed to extract the archive' ) ; } let commitHash = defaultBranch ; const lastDashIndex = archiveDir . lastIndexOf ( '-' ) ; if ( lastDashIndex !== - 1 ) { commitHash = archiveDir . substr ( lastDashIndex + 1 ) ; } logger . info ( `Using commit hash "${ commitHash }"` ) ; const files = await getFiles ( archiveDir , [ ] ) ; let mismatchedTags = false ; const failureMessages = [ ] ; for ( const file of files ) { if ( configuration . ignoredFile ( file ) ) { logger . info ( 'ignoring file from configuration: ' + file ) ; continue ; } try { const fileContents = await pfs . readFile ( file , 'utf-8' ) ; const parseResult = parseRegionTags ( fileContents , file . replace ( archiveDir + '/' , '' ) , owner , repo , commitHash ) ; if ( ! parseResult . result ) { mismatchedTags = true ; for ( const violation of parseResult . violations ) { const formatted = formatMatchingViolation ( violation ) ; failureMessages . push ( `- [ ] ${ formatted }` ) ; } } } catch ( e ) { const err = e as Error ; err . message = `Failed to read the file: ${ err . message }` ; logger . error ( err ) ; continue ; } } let bodyDetail = 'Great job! No unmatching region tags found!' ; if ( mismatchedTags ) { bodyDetail = failureMessages . join ( '\n' ) ; } await context . octokit . issues . update ( { owner : owner , repo : repo , issue_number : issueNumber , body : formatBody ( context . payload . issue . body as string , commentMark , `## snippet-bot scan result
Life is too short to manually check unmatched region tags.
Here is the result:
${ bodyDetail }` ) , } ) ; } catch ( e ) { const err = e as Error ; err . message = `Failed to scan files: ${ err . message }` ; logger . error ( err ) ; await context . octokit . issues . update ( { owner : owner , repo : repo , issue_number : issueNumber , body : formatBody ( context . payload . issue . body as string , commentMark , `## snippet-bot scan result\nFailed running the full scan: ${ err }.` ) , } ) ; } finally { await pfs . rmdir ( tmpDir . name , { recursive : true } ) ; } } async function scanPullRequest ( context , pull_request , configuration , refreshing = false ) { const installationId = context . payload . installation ?. id ; const owner = context . payload . repository . owner . login ; const repo = context . payload . repository . name ; const aggregator = new CheckAggregator ( context . octokit , 'snippet-bot check' , configuration . aggregateChecks ( ) ) ; const result = await parseRegionTagsInPullRequest ( context . octokit , pull_request . diff_url , pull_request . base . repo . owner . login , pull_request . base . repo . name , pull_request . base . sha , pull_request . head . repo . owner . login , pull_request . head . repo . name , pull_request . head . sha ) ; let mismatchedTags = false ; let tagsFound = false ; const failureMessages = [ ] ; const noPrefixReq = pull_request . labels . some ( ( label ) => { return label . name === NO_PREFIX_REQ_LABEL ; } ) ; const parseResults = new Map < string , ParseResult > ( ) ; for ( const file of result . files ) { if ( configuration . ignoredFile ( file ) ) { logger . info ( 'ignoring file from configuration: ' + file ) ; continue ; } try { const blob = await context . octokit . repos . getContent ( { owner : pull_request . head . repo . owner . login , repo : pull_request . head . repo . name , path : file , ref : pull_request . head . sha , } ) ; if ( ! isFile ( blob . data ) ) { continue ; } const fileContents = Buffer . from ( blob . data . content , 'base64' ) . toString ( 'utf8' ) ; const parseResult = parseRegionTags ( fileContents , file , owner , repo , pull_request . head . sha ) ; parseResults . set ( file , parseResult ) ; if ( ! parseResult . result ) { mismatchedTags = true ; for ( const violation of parseResult . violations ) { failureMessages . push ( formatMatchingViolation ( violation ) ) ; } } if ( parseResult . tagsFound ) { tagsFound = true ; } } catch ( e ) { const err = e as RequestError & Error ; if ( err . status === 403 || err . status === 404 ) { logger . info ( `ignoring 403/404 errors upon fetching ${ file }: ${ err . message }` ) ; } else { throw err ; } } } const checkParams = context . repo ( { name : 'Mismatched region tag' , conclusion : 'success' as Conclusion , head_sha : pull_request . head . sha , output : { title : 'Region tag check' , summary : 'Region tag successful' , text : 'Region tag successful' , } , } ) ; if ( mismatchedTags ) { checkParams . conclusion = 'failure' ; checkParams . output = { title : 'Mismatched region tag detected.' , summary : 'Some new files have mismatched region tag' , text : failureMessages . join ( '\n' ) , } ; } if ( configuration . alwaysCreateStatusCheck ( ) || configuration . aggregateChecks ( ) || tagsFound ) { await aggregator . add ( checkParams ) ; } let commentBody = '' ; if ( result . changes . length === 0 ) { if ( ! refreshing && ! configuration . alwaysCreateStatusCheck ( ) && ! configuration . aggregateChecks ( ) ) { return ; } commentBody += 'No region tags are edited in this PR.\n' ; } const prNumber = pull_request . number ; let productPrefixViolations = [ ] ; if ( ! noPrefixReq ) { productPrefixViolations = await checkProductPrefixViolations ( result , configuration ) ; } const removingUsedTagsViolations = await checkRemovingUsedTagViolations ( result , configuration , parseResults , pull_request . base . repo . full_name , pull_request . base . ref ) ; const removeUsedTagViolations = [ ... ( removingUsedTagsViolations . get ( 'REMOVE_USED_TAG' ) as Violation [ ] ) , ... ( removingUsedTagsViolations . get ( 'REMOVE_CONFLICTING_TAG' ) as Violation [ ] ) , ] ; const removeSampleBrowserViolations = removingUsedTagsViolations . get ( 'REMOVE_SAMPLE_BROWSER_PAGE' ) as Violation [ ] ; const removeFrozenRegionTagViolations = removingUsedTagsViolations . get ( 'REMOVE_FROZEN_REGION_TAG' ) as Violation [ ] ; const prefixCheckParams = context . repo ( { name : 'Region tag product prefix' , conclusion : 'success' as Conclusion , head_sha : pull_request . head . sha , output : { title : 'No violations' , summary : 'No violations found' , text : 'All the tags have appropriate product prefix' , } , } ) ; const removeUsedTagCheckParams = context . repo ( { name : 'Disruptive region tag removal' , conclusion : 'success' as Conclusion , head_sha : pull_request . head . sha , output : { title : 'No violations' , summary : 'No violations found' , text : 'No disruptive region tag removal' , } , } ) ; if ( productPrefixViolations . length > 0 || removeUsedTagViolations . length > 0 ) { commentBody += 'Here is the summary of possible violations 😱' ; if ( productPrefixViolations . length > 0 ) { let summary = '' ; if ( productPrefixViolations . length === 1 ) { summary = 'There is a possible violation for not having product prefix.' ; } else { summary = `There are ${ productPrefixViolations . length } possible violations for not having product prefix.` ; } const productPrefixViolationsDetail = formatViolations ( productPrefixViolations , summary ) ; commentBody += productPrefixViolationsDetail ; prefixCheckParams . conclusion = 'failure' ; prefixCheckParams . output = { title : 'Missing region tag prefix' , summary : 'Some region tags do not have appropriate prefix' , text : productPrefixViolationsDetail , } ; } if ( removeUsedTagViolations . length > 0 ) { let summary = '' ; if ( removeUsedTagViolations . length === 1 ) { summary = 'There is a possible violation for removing region tag in use.' ; } else { summary = `There are ${ removeUsedTagViolations . length } possible violations for removing region tag in use.` ; } const removeUsedTagViolationsDetail = formatViolations ( removeUsedTagViolations , summary ) ; commentBody += removeUsedTagViolationsDetail ; removeUsedTagCheckParams . conclusion = 'failure' ; removeUsedTagCheckParams . output = { title : 'Removal of region tags in use' , summary : '' , text : removeUsedTagViolationsDetail , } ; } commentBody += '**The end of the violation section. All the stuff below is FYI purposes only.**\n\n' ; commentBody += '---\n' ; } if ( removeSampleBrowserViolations . length > 0 ) { let summary = 'You are about to delete the following sample browser page' ; if ( removeSampleBrowserViolations . length > 1 ) { summary += 's' ; } summary += '.' ; commentBody += formatViolations ( removeSampleBrowserViolations , summary ) ; commentBody += '---\n' ; } if ( removeFrozenRegionTagViolations . length > 0 ) { let summary = 'You are about to delete the following frozen region tag' ; if ( removeFrozenRegionTagViolations . length > 1 ) { summary += 's' ; } summary += '.' ; commentBody += formatViolations ( removeFrozenRegionTagViolations , summary ) ; commentBody += '---\n' ; } if ( result . added > 0 || result . deleted > 0 ) { commentBody += 'Here is the summary of changes.\n' ; } if ( result . added > 0 ) { const plural = result . added === 1 ? '' : 's' ; const summary = `You are about to add ${ result . added } region tag${ plural }.` ; let detail = '' ; for ( const change of result . changes ) { if ( change . type === 'add' ) { detail += `- ${ formatRegionTag ( change ) }\n` ; } } commentBody += formatExpandable ( summary , detail ) ; } if ( result . deleted > 0 ) { const plural = result . deleted === 1 ? '' : 's' ; const summary = `You are about to delete ${ result . deleted } region tag${ plural }.\n` ; let detail = '' ; for ( const change of result . changes ) { if ( change . type === 'del' ) { detail += `- ${ formatRegionTag ( change ) }\n` ; } } commentBody += formatExpandable ( summary , detail ) ; } if ( commentBody . length > MAX_CHARS_IN_COMMENT ) { commentBody = commentBody . substring ( 0 , MAX_CHARS_IN_COMMENT ) ; const newLineIndex = commentBody . lastIndexOf ( '\n' ) ; if ( newLineIndex !== - 1 ) { commentBody = commentBody . substring ( 0 , newLineIndex ) ; } commentBody += '\n...(The comment is too long, omitted)\n' ; } commentBody += `---
This comment is generated by [snippet-bot](https://github.com/apps/snippet-bot).
If you find problems with this result, please file an issue at:
https://github.com/googleapis/repo-automation-bots/issues.
To update this comment, add \`${ REFRESH_LABEL }\` label or use the checkbox below:
${ REFRESH_UI }
` ; const onlyUpdate = result . changes . length === 0 ; await addOrUpdateIssueComment ( context . octokit , owner , repo , prNumber , installationId as number , commentBody , onlyUpdate ) ; if ( configuration . alwaysCreateStatusCheck ( ) || configuration . aggregateChecks ( ) || productPrefixViolations . length > 0 ) { await aggregator . add ( prefixCheckParams ) ; } if ( configuration . alwaysCreateStatusCheck ( ) || configuration . aggregateChecks ( ) || removeUsedTagViolations . length > 0 ) { await aggregator . add ( removeUsedTagCheckParams ) ; } await aggregator . submit ( ) ; logger . metric ( 'snippet-bot-violations' , { target : pull_request . url , violation_type : 'UNMATCHED_REGION_TAG' , count : failureMessages . length , } ) ; logger . metric ( 'snippet-bot-violations' , { target : pull_request . url , violation_type : 'MISSING_PRODUCT_PREFIX' , count : productPrefixViolations . length , } ) ; logger . metric ( 'snippet-bot-violations' , { target : pull_request . url , violation_type : 'REMOVING_USED_TAG' , count : removeUsedTagViolations . length , } ) ; } /**
 * Creates a comment mark used for addOrupdateissuecomment.
 * I'll move this function to gcf-utils later.
 */ function getCommentMark ( installationId ) { return `<!-- probot comment [${ installationId }]-->` ; } export = ( app ) => { app . on ( 'schedule.repository' as any , async ( context ) => { const owner = context . payload . organization . login ; const repo = context . payload . repository . name ; const configOptions = await getConfig < ConfigurationOptions > ( context . octokit , owner , repo , CONFIGURATION_FILE_PATH , { schema : schema } ) ; if ( configOptions === null ) { logger . info ( `snippet-bot is not configured for ${ owner }/${ repo }.` ) ; return ; } await syncLabels ( context . octokit , owner , repo , SNIPPET_BOT_LABELS ) ; } ) ; app . on ( 'issue_comment.edited' , async ( context ) => { const commentMark = getCommentMark ( context . payload . installation ?. id ) ; if ( ! context . payload . comment . body . includes ( commentMark ) || ! context . payload . comment . body . includes ( REFRESH_STRING ) ) { return ; } const repoUrl = context . payload . repository . full_name ; const { owner , repo } = context . repo ( ) ; const configOptions = await getConfig < ConfigurationOptions > ( context . octokit , owner , repo , CONFIGURATION_FILE_PATH , { schema : schema } ) ; if ( configOptions === null ) { logger . info ( `snippet-bot is not configured for ${ repoUrl }.` ) ; return ; } const configuration = new Configuration ( { ... DEFAULT_CONFIGURATION , ... configOptions , } ) ; logger . info ( { config : configuration } ) ; const prNumber = context . payload . issue . number ; const prResponse = await context . octokit . pulls . get ( { owner : owner , repo : repo , pull_number : prNumber , } ) ; invalidateCache ( ) ; await scanPullRequest ( context , prResponse . data as PullRequest , configuration , true ) ; } ) ; app . on ( [ 'issues.opened' , 'issues.reopened' ] , async ( context ) => { const repoUrl = context . payload . repository . full_name ; const { owner , repo } = context . repo ( ) ; const configOptions = await getConfig < ConfigurationOptions > ( context . octokit , owner , repo , CONFIGURATION_FILE_PATH , { schema : schema } ) ; if ( configOptions === null ) { logger . info ( `snippet-bot is not configured for ${ repoUrl }.` ) ; return ; } const configuration = new Configuration ( { ... DEFAULT_CONFIGURATION , ... configOptions , } ) ; logger . info ( { config : configuration } ) ; await fullScan ( context , configuration ) ; } ) ; app . on ( 'pull_request.labeled' , async ( context ) => { const repoUrl = context . payload . repository . full_name ; const { owner , repo } = context . repo ( ) ; const configOptions = await getConfig < ConfigurationOptions > ( context . octokit , owner , repo , CONFIGURATION_FILE_PATH , { schema : schema } ) ; if ( configOptions === null ) { logger . info ( `snippet-bot is not configured for ${ repoUrl }.` ) ; return ; } const configuration = new Configuration ( { ... DEFAULT_CONFIGURATION , ... configOptions , } ) ; logger . info ( { config : configuration } ) ; if ( context . payload . pull_request . labels === undefined ) { return ; } const labelFound = context . payload . pull_request . labels . some ( ( label ) => { return label . name === REFRESH_LABEL ; } ) ; if ( ! labelFound ) { return ; } try { await context . octokit . issues . removeLabel ( context . issue ( { name : REFRESH_LABEL } ) ) ; } catch ( e ) { const err = e as RequestError ; if ( err . status !== 404 ) { throw err ; } } invalidateCache ( ) ; await scanPullRequest ( context , context . payload . pull_request as PullRequest , configuration , true ) ; } ) ; app . on ( [ 'pull_request.opened' , 'pull_request.reopened' , 'pull_request.edited' , 'pull_request.synchronize' , ] , async ( context ) => { if ( context . payload . pull_request . state === 'closed' ) { logger . info ( `The pull request ${ context . payload . pull_request . url } is closed, exiting.` ) ; return ; } if ( context . payload . pull_request . head . repo === undefined || context . payload . pull_request . head . repo === null ) { logger . info ( `The head repo is undefined for ${ context . payload . pull_request . url }, exiting.` ) ; return ; } const repoUrl = context . payload . repository . full_name ; const { owner , repo } = context . repo ( ) ; const configChecker = new ConfigChecker < ConfigurationOptions > ( schema , CONFIGURATION_FILE_PATH ) ; await configChecker . validateConfigChanges ( context . octokit , owner , repo , context . payload . pull_request . head . sha , context . payload . pull_request . number ) ; const configOptions = await getConfig < ConfigurationOptions > ( context . octokit , owner , repo , CONFIGURATION_FILE_PATH , { schema : schema } ) ; if ( configOptions === null ) { logger . info ( `snippet-bot is not configured for ${ repoUrl }.` ) ; return ; } const configuration = new Configuration ( { ... DEFAULT_CONFIGURATION , ... configOptions , } ) ; logger . info ( { config : configuration } ) ; await scanPullRequest ( context , context . payload . pull_request as PullRequest , configuration ) ; } ) ; } ;