import cheerio from "cheerio" ; import qs from "querystring" ; import fetch , { Response as FetchResponse } from "node-fetch" ; import { Ad } from "../ad" ; import { BANNED , HTML_REQUEST_HEADERS , POSSIBLE_BAD_MARKUP } from "../constants" ; import { AdInfo } from "../scraper" ; import { PageResults , ResolvedSearchParameters } from "../search" ; const KIJIJI_BASE_URL = "https://www.kijiji.ca" ; const KIJIJI_SEARCH_URL = KIJIJI_BASE_URL + "/b-search.html" ; const IMG_REGEX = /\/s\-l\d+\.jpg$/ ; const LOCATION_REGEX = /(.+)(\/.*)$/ ; function dateFromRelativeDateString ( dateString ) { if ( dateString ) { dateString = dateString . toLowerCase ( ) . replace ( /\//g , " " ) ; const split = dateString . split ( " " ) ; const d = new Date ( ) ; if ( split . length === 3 ) { d . setHours ( 0 , 0 , 0 , 0 ) ; d . setDate ( parseInt ( split [ 0 ] ) ) ; d . setMonth ( parseInt ( split [ 1 ] ) - 1 ) ; d . setFullYear ( parseInt ( split [ 2 ] ) ) ; return d ; } else if ( split . length === 4 ) { const num = parseInt ( split [ 1 ] ) ; const timeUnit = split [ 2 ] ; if ( timeUnit === "minutes" ) { d . setMinutes ( d . getMinutes ( ) - num ) ; d . setSeconds ( 0 , 0 ) ; } else if ( timeUnit === "hours" ) { d . setHours ( d . getHours ( ) - num , 0 , 0 , 0 ) ; } return d ; } else if ( dateString == "yesterday" ) { d . setDate ( d . getDate ( ) - 1 ) ; d . setHours ( 0 , 0 , 0 , 0 ) ; return d ; } } return new Date ( NaN ) ; } function parseResultsHTML ( html ) { const adResults = [ ] ; const $ = cheerio . load ( html ) ; const allAdElements = $ ( ".regular-ad" ) ; const filteredAdElements = allAdElements . not ( ".third-party" ) ; filteredAdElements . each ( ( _i , item ) => { const path = $ ( item ) . find ( "a.title" ) . attr ( "href" ) ; const url = KIJIJI_BASE_URL + path ; const info = { id : $ ( item ) . data ( "listing-id" ) ?. toString ( ) || "" , title : $ ( item ) . find ( "a.title" ) . text ( ) . trim ( ) , image : ( $ ( item ) . find ( ".image img" ) . data ( "src" ) || $ ( item ) . find ( ".image img" ) . attr ( "src" ) || "" ) . replace ( IMG_REGEX , "/s-l2000.jpg" ) , date : dateFromRelativeDateString ( ( $ ( item ) . find ( ".date-posted" ) . text ( ) || $ ( item ) . find ( ".posted" ) . text ( ) ) . trim ( ) . split ( "\n" ) [ 0 ] ) , description : ( $ ( item ) . find ( ".description > p" ) . text ( ) || $ ( item ) . find ( ".description" ) . text ( ) ) . trim ( ) } ; if ( ! path ) { throw new Error ( `Result ad has no URL. ${ POSSIBLE_BAD_MARKUP }` ) ; } adResults . push ( new Ad ( url , info ) ) ; } ) ; return adResults ; } /**
 * Searcher implementation
 */ export class HTMLSearcher { private firstResultPageURL : string | undefined = undefined ; private async getFirstResultPageURL ( params ) { if ( this . firstResultPageURL === undefined ) { const res = await fetch ( `${ KIJIJI_SEARCH_URL }?${ qs . stringify ( params ) }` , { headers : HTML_REQUEST_HEADERS } ) ; if ( res . status === 403 ) { throw new Error ( BANNED ) ; } else if ( res . status !== 200 || ! res . url ) { throw new Error ( `Kijiji failed to redirect to results page. ${ POSSIBLE_BAD_MARKUP }` ) ; } this . firstResultPageURL = res . url ; } return this . firstResultPageURL ; } async getPageResults ( params , pageNum ) { const firstResultPageURL = await this . getFirstResultPageURL ( params ) ; const url = firstResultPageURL . replace ( LOCATION_REGEX , `$1/page-${ pageNum }$2` ) ; return fetch ( url , { headers : HTML_REQUEST_HEADERS } ) . then ( res => { if ( res . status === 403 ) { throw new Error ( BANNED ) ; } return res . text ( ) ; } ) . then ( body => ( { pageResults : parseResultsHTML ( body ) , isLastPage : body . indexOf ( '"isLastPage":true' ) !== - 1 } ) ) ; } }