import { OctokitType } from './utils/octokit-util' ; import { Storage } from '@google-cloud/storage' ; import { logger } from 'gcf-utils' ; /**
 * This interface is used for comforting typescript's type system to
 * deal with the response from `octokit.repos.getContent`.
 */ interface File { content : string | undefined ; } /**
 * This function is used for comforting typescript's type system to
 * deal with the response from `octokit.repos.getContent`.
 */ function isFile ( file ) { return ( file as File ) . content !== undefined ; } const COMMIT_CACHE_BUCKET = 'github_commit_cache' ; export class FileIterator { octokit : OctokitType ; owner : string ; repo : string ; storage : Storage ; lastSha ? : string ; constructor ( owner , repo , octokit ) { this . octokit = octokit ; this . owner = owner ; this . repo = repo ; this . storage = new Storage ( ) ; } async * repoMetadata ( ) { const files = await this . getFileListing ( ) ; if ( ! files ) { logger . warn ( `unable to find file listing for ${ this . owner }/${ this . repo }` ) ; return ; } for ( const path of files . trim ( ) . split ( /\r?\n/ ) ) { if ( path . toLowerCase ( ) . endsWith ( '.repo-metadata.json' ) ) { const resp = ( await this . octokit . rest . repos . getContent ( { owner : this . owner , repo : this . repo , path , } ) ) . data ; if ( isFile ( resp ) ) { yield [ path , Buffer . from ( resp . content , 'base64' ) . toString ( 'utf8' ) ] ; } } } } async getFileListing ( ) { const perPage = 5 ; const commits = ( await this . octokit . rest . repos . listCommits ( { owner : this . owner , repo : this . repo , per_page : perPage , } ) ) . data ; let manifest = '' ; for ( let i = 0 ; i < perPage ; i ++ ) { const commit = commits [ i ] ; if ( commit ) { const manifestFile = `owners/${ this . owner }/repos/${ this . repo }/commits/${ commit . sha }/file_manifest.txt` ; logger . info ( `get ${ manifestFile } from ${ COMMIT_CACHE_BUCKET }` ) ; try { const [ result ] = await this . storage . bucket ( COMMIT_CACHE_BUCKET ) . file ( manifestFile ) . download ( ) ; this . lastSha = commit . sha ; manifest = result . toString ( 'utf8' ) ; break ; } catch ( _err ) { const err = _err as { code : number ; message : string ; } ; if ( err . code === 404 ) { continue ; } else { logger . error ( `received error code = ${ err . code } msg = ${ err . message }` , err ) ; throw err ; } } } } return manifest ; } async getFile ( file ) { if ( ! this . lastSha ) { await this . getFileListing ( ) ; } file = `owners/${ this . owner }/repos/${ this . repo }/commits/${ this . lastSha }/files/${ file }` ; const [ result ] = await this . storage . bucket ( COMMIT_CACHE_BUCKET ) . file ( file ) . download ( ) ; return result . toString ( 'utf8' ) ; } }