import fetch from "node-fetch" ; import cheerio from "cheerio" ; import { BANNED , HTML_REQUEST_HEADERS } from "../constants" ; import { cleanAdDescription , getLargeImageURL , isNumber } from "../helpers" ; import { AdInfo } from "../scraper" ; function castAttributeValue ( attr ) { let value = attr . machineValue ; if ( typeof value !== "string" ) { return undefined ; } value = value . trim ( ) ; const localizedValue = ( attr . localeSpecificValues ?. en ?. value || "" ) . toLowerCase ( ) ; if ( value . toLowerCase ( ) === "true" ) { return true ; } else if ( value . toLowerCase ( ) === "false" ) { return false ; } else if ( isNumber ( value ) ) { if ( isNumber ( localizedValue ) ) { return Number ( localizedValue ) ; } return Number ( value ) ; } else if ( ! isNaN ( Date . parse ( value ) ) && isNumber ( value [ 0 ] ) ) { return new Date ( value ) ; } else { return value ; } } function parseResponseHTML ( html ) { const info = new AdInfo ( ) ; const $ = cheerio . load ( html ) ; let adData = { } ; let json = $ ( "#FesLoader > script" ) . text ( ) . replace ( "window.__data=" , "" ) ; json = json . substring ( 0 , json . length - 1 ) ; if ( json . length === 0 || Object . keys ( adData = JSON . parse ( json ) ) . length === 0 || ! adData . hasOwnProperty ( "config" ) || ! adData . config . hasOwnProperty ( "adInfo" ) || ! adData . config . hasOwnProperty ( "VIP" ) ) { return null ; } adData = adData . config ; const adId = adData . VIP . adId ; const adTitle = adData . adInfo . title ; const adDateMs = adData . VIP . sortingDate ; if ( adId === undefined || adTitle === undefined || adDateMs === undefined ) { return null ; } info . id = adId . toString ( ) ; info . title = adTitle ; info . description = cleanAdDescription ( adData . VIP . description || "" ) ; info . date = new Date ( adDateMs ) ; info . image = getLargeImageURL ( adData . adInfo . sharingImageUrl || "" ) ; ( adData . VIP . media || [ ] ) . forEach ( ( m ) => { if ( m . type === "image" && m . href && typeof m . href === "string" ) { info . images . push ( getLargeImageURL ( m . href ) ) ; } } ) ; ( adData . VIP . adAttributes || [ ] ) . forEach ( ( a ) => { const name = a . machineKey ; const value = castAttributeValue ( a ) ; if ( typeof name === "string" && value !== undefined ) { info . attributes [ name ] = value ; } } ) ; if ( adData . VIP . price && typeof adData . VIP . price . amount === "number" ) { info . attributes [ "price" ] = adData . VIP . price . amount / 100.0 ; } if ( adData . VIP . adLocation ) { info . attributes [ "location" ] = adData . VIP . adLocation ; } if ( adData . VIP . adType ) { info . attributes [ "type" ] = adData . VIP . adType ; } if ( adData . VIP . visitCounter ) { info . attributes [ "visits" ] = adData . VIP . visitCounter ; } return info ; } export function scrapeHTML ( url ) { return fetch ( url , { headers : HTML_REQUEST_HEADERS } ) . then ( res => { if ( res . status === 403 ) { throw new Error ( BANNED ) ; } return res . text ( ) ; } ) . then ( body => { return parseResponseHTML ( body ) ; } ) ; }