import { GlobalLogger } from 'logger' ; import { inject , injectable } from 'inversify' ; import { Crawler , CrawlerRunOptions , crawlerIocTypes } from 'accessibility-insights-crawler' ; import { Page } from 'puppeteer' ; import { BatchConfig } from 'azure-services' ; type CrawlerProvider = ( ) => Promise < Crawler < string [ ] > > ; @ injectable ( ) export class CrawlRunner { private readonly storageDirName = 'crawler_storage' ; constructor ( @ inject ( crawlerIocTypes . CrawlerProvider ) private readonly getCrawler , @ inject ( GlobalLogger ) private readonly logger , @ inject ( BatchConfig ) private readonly batchConfig ) { } public async run ( baseUrl , discoveryPatterns , page ) { const crawler = await this . getCrawler ( ) ; if ( crawler == null ) { this . logger . logInfo ( 'No crawler instance created by crawler provider.' ) ; return undefined ; } this . logger . logInfo ( 'Starting web page crawling.' ) ; let result ; try { const commonOptions = await this . getCommonCrawlOptions ( ) ; const crawlerRunOptions = { baseUrl , discoveryPatterns , baseCrawlPage : page , ... commonOptions , } ; result = await crawler . crawl ( crawlerRunOptions ) ; } catch ( ex ) { this . logger . logError ( 'Failure while crawling web page.' , { error : JSON . stringify ( ex ) } ) ; return undefined ; } this . logger . logInfo ( `Web page crawling completed successfully. Found ${ result ? result . length : 0 } urls.` ) ; return result ; } private async getCommonCrawlOptions ( ) { const outputDir = `${ this . batchConfig . taskWorkingDir }/${ this . storageDirName }` ; return { maxRequestsPerCrawl : 1000 , localOutputDir : outputDir , silentMode : true , restartCrawl : true , } ; } }