import moment from 'moment-timezone' ; import { Location } from '../index' ; import { Page } from 'puppeteer' ; import countries from 'i18n-iso-countries' ; import cities from 'all-the-cities' ; export const getIsCountry = ( text ) : boolean => { const countriesList = Object . values ( countries . getNames ( 'en' ) ) ; const lowerCaseText = text . toLowerCase ( ) ; if ( [ 'united states' , 'the netherlands' ] . includes ( lowerCaseText ) ) { return true ; } return ! ! countriesList . find ( country => country . toLowerCase ( ) === lowerCaseText ) ; } ; export const getIsCity = ( text ) : boolean => { const lowerCaseText = text . toLowerCase ( ) ; if ( [ 'new york' ] . includes ( lowerCaseText ) ) { return true ; } return ! ! cities . find ( city => city . name . toLowerCase ( ) === lowerCaseText ) ; } ; export const formatDate = ( date ) : string => { if ( date === 'Present' ) { return moment ( ) . format ( ) ; } return moment ( date , 'MMMY' ) . format ( ) ; } ; export const getDurationInDays = ( formattedStartDate , formattedEndDate ) : number | null => { if ( ! formattedStartDate || ! formattedEndDate ) return null ; return moment ( formattedEndDate ) . diff ( moment ( formattedStartDate ) , 'days' ) + 1 ; } ; export const getLocationFromText = ( text ) : Location | null => { if ( ! text ) return null ; const cleanText = text . replace ( ' Area' , '' ) . trim ( ) ; const parts = cleanText . split ( ', ' ) ; let city = null ; let province = null ; let country = null ; if ( parts . length === 3 ) { city = parts [ 0 ] ; province = parts [ 1 ] ; country = parts [ 2 ] ; return { city , province , country } ; } if ( parts . length === 2 ) { if ( getIsCity ( parts [ 0 ] ) && getIsCountry ( parts [ 1 ] ) ) { return { city : parts [ 0 ] , province , country : parts [ 1 ] } ; } if ( getIsCity ( parts [ 0 ] ) && ! getIsCountry ( parts [ 1 ] ) ) { return { city : parts [ 0 ] , province : parts [ 1 ] , country } ; } return { city , province : parts [ 0 ] , country : parts [ 1 ] } ; } if ( getIsCountry ( parts [ 0 ] ) ) { return { city , province , country : parts [ 0 ] } ; } if ( getIsCity ( parts [ 0 ] ) ) { return { city : parts [ 0 ] , province , country } ; } return { city , province : parts [ 0 ] , country } ; } ; export const getCleanText = ( text ) => { const regexRemoveMultipleSpaces = / +/g ; const regexRemoveLineBreaks = /(\r\n\t|\n|\r\t)/gm ; if ( ! text ) return null ; const cleanText = text . replace ( regexRemoveLineBreaks , '' ) . replace ( regexRemoveMultipleSpaces , ' ' ) . replace ( '...' , '' ) . replace ( 'See more' , '' ) . replace ( 'See less' , '' ) . trim ( ) ; return cleanText ; } ; export const statusLog = ( section , message , scraperSessionId ? ) => { const sessionPart = ( scraperSessionId ) ? ` (${ scraperSessionId })` : '' ; const messagePart = ( message ) ? `: ${ message }` : null ; return console . log ( `Scraper (${ section })${ sessionPart }${ messagePart }` ) ; } ; export const autoScroll = async ( page ) => { await page . evaluate ( async ( ) => { await new Promise ( ( resolve , reject ) => { var totalHeight = 0 ; var distance = 500 ; var timer = setInterval ( ( ) => { var scrollHeight = document . body . scrollHeight ; window . scrollBy ( 0 , distance ) ; totalHeight += distance ; if ( totalHeight >= scrollHeight ) { clearInterval ( timer ) ; resolve ( ) ; } } , 100 ) ; } ) ; } ) ; } ; export const getHostname = ( url ) => { return new URL ( url ) . hostname ; } ;