import puppeteer from 'puppeteer' ; import progress from 'cli-progress' ; import dotenv from 'dotenv' ; dotenv . config ( ) ; async function scrape ( domain , path ) { const browser = await puppeteer . launch ( { headless : false } ) ; const page = await browser . newPage ( ) ; await page . setViewport ( { width : 1600 , height : 800 } ) ; await page . goto ( `${ domain }${ path }` ) ; await page . waitForFunction ( 'window.restaurants !== undefined' ) ; await new Promise ( ( resolve ) => setTimeout ( resolve , 20000 ) ) ; const restaurants = await page . evaluate ( ( ) => { const properties = Object . keys ( window ) . filter ( ( prop ) => /^[A-Z_]+$/g . test ( prop ) ) . reduce ( ( constants , prop ) => { if ( typeof window [ prop ] !== 'number' ) return constants ; constants [ window [ prop ] ] = `${ prop }` . toLowerCase ( ) ; return constants ; } , new Array ( 39 ) . fill ( null ) ) ; const restaurants = window [ 'restaurants' ] . reduce ( ( restaurants , r ) => { const restaurant = Object . keys ( r ) . reduce ( ( restaurant , prop ) => { if ( ! properties [ prop ] ) return restaurant ; restaurant [ properties [ prop ] ] = r [ prop ] ; return restaurant ; } , { } ) ; restaurants . push ( restaurant ) ; return restaurants ; } , [ ] ) ; return restaurants ; } ) ; const bar = new progress . SingleBar ( { } , progress . Presets . shades_classic ) ; const total = restaurants . length ; bar . start ( total , 0 ) ; await ( async function loop ( ) { if ( ! restaurants . length ) { bar . stop ( ) ; browser . close ( ) ; return ; } const restaurant = restaurants . pop ( ) ; bar . update ( total - restaurants . length ) ; const { url , branch , logo , categories } = restaurant . restaurant_info_and_links ; delete restaurant . restaurant_info_and_links ; restaurant . url = `${ domain }${ url }` ; restaurant . branch = branch ; restaurant . logo = logo ; restaurant . description = categories ; await page . goto ( restaurant . url ) ; const additionalInfo = JSON . parse ( await page . evaluate ( ( ) => document . querySelector ( 'script[type="application/ld+json"]' ) ?. innerHTML || '{}' ) ) ; const currentRestaurant = await page . evaluate ( ( ) => window [ 'currentRestaurant' ] ) ; const completeRestaurant = { ... restaurant , ... Object . keys ( currentRestaurant ) . reduce ( ( restaurant , key ) => { if ( [ 'Times' , 'Urls' , 'Translations' , 'Locations' , 'Polygons' ] . includes ( key ) ) { return restaurant ; } restaurant [ key [ 0 ] . toLowerCase ( ) + key . substring ( 1 ) ] = currentRestaurant [ key ] ; return restaurant ; } , { } ) , address : additionalInfo . address , aggregateRating : additionalInfo . aggregateRating , geo : additionalInfo . geo , image : additionalInfo . image } ; } ) ( ) ; }