import { Ad } from "./ad" ; import { APISearcher } from "./backends/api-searcher" ; import { HTMLSearcher } from "./backends/html-searcher" ; import { getScraperOptions , sleep , ScraperOptions , ScraperType } from "./helpers" ; type KijijiIdTreeNode = { id : number ; } ; /**
 * Kijiji ad search parameters
 */ export type SearchParameters = { /**
     * Id of the geographical location to search in
     */ locationId ? : number | KijijiIdTreeNode ; /**
     * Id of the ad category to search in
     */ categoryId ? : number | KijijiIdTreeNode ; /**
     * The minimum desired price of the results
     */ minPrice ? : number ; /**
     * The maximum desired price of the results
     */ maxPrice ? : number ; /**
     * Type of ad to return. Leave undefined for both
     */ adType ? : string ; /**
     * Other parameters, specific to the category. Use browser developer
     * tools when performing a specific search to find more parameters.
     * After submitting your search on Kijiji or updating the filter being
     * applied, examine the request for https://www.kijiji.ca/b-search.html.
     * Any parameter used in the query string for that request is able to be
     * specified in here.
     */ [ paramName ] : any ; } ; export type ResolvedSearchParameters = { locationId : number ; categoryId : number ; [ paramName ] : any ; } ; /**
 * Parameters that control the behavior of the scraper
 */ export type SearchOptions = { /**
     * Amount of time in milliseconds to wait between scraping each result page.
     * This is useful to avoid detection and bans from Kijiji. Defaults to 1000.
     */ pageDelayMs ? : number ; /**
     * Minimum number of ads to fetch (if available). Note that Kijiji results
     * are returned in pages of up to `20` ads, so if you set this to something
     * like `29`, up to `40` results may be retrieved. A negative value indicates
     * no limit (retrieve as many ads as possible).
     */ minResults ? : number ; /**
     * Maximum number of ads to return. This simply removes excess results from
     * the array that is returned (i.e., if `minResults` is `40` and `maxResults`
     * is `7`, `40` results will be fetched from Kijiji and the last `33` will be
     * discarded). A negative value indicates no limit.
     */ maxResults ? : number ; /**
     * When using the HTML scraper, the details of each query result are scraped in
     * separate, subsequent requests by default. To suppress this behavior and return
     * only the data retrieved by the initial query, set this option to `false`. Note
     * that ads will lack some information if you do this and `Ad.isScraped()` will
     * will return `false` until `Ad.scrape()` is called to retrieve the missing
     * information. This option does nothing when using the API scraper (default)
     */ scrapeResultDetails ? : boolean ; /**
     * When `scrapeResultDetails` is `true`, the amount of time in milliseconds to
     * wait in between each request for result details. A value of 0 will cause all
     * such requests to be made at the same time. This is useful to avoid detection
     * and bans from Kijiji. Defaults to 500.
     */ resultDetailsDelayMs ? : number ; } ; /**
 * The search results for one page
 */ export type PageResults = { /**
     * Ads from the result page
     */ pageResults : Ad [ ] ; /**
     * Whether or not this page is the last page of results
     */ isLastPage : boolean ; } ; /**
 * Generic interface for a Kijiji searcher
 */ export interface Searcher { param params param pageNum returns getPageResults ( params , pageNum ) : Promise < PageResults > ; } ; async function getSearchResults ( searcher , params , options ) { const results = [ ] ; let pageNum = 1 ; try { let needResults = options . minResults !== 0 ; while ( needResults ) { const { pageResults , isLastPage } = await searcher . getPageResults ( params , pageNum ++ ) ; results . push ( ... pageResults ) ; needResults = pageResults . length > 0 && ! isLastPage && ( results . length < options . minResults || options . minResults < 0 ) ; if ( needResults ) { await sleep ( options . pageDelayMs ) ; } } } catch ( err ) { throw new Error ( `Error parsing Kijiji search results: ${ err . message }` ) ; } return results ; } function ensureIntProp ( obj , propName ) { if ( ! obj . hasOwnProperty ( propName ) || ! Number . isInteger ( obj [ propName ] ) ) { throw new Error ( `Integer property '${ propName }' must be specified` ) ; } } function getSearchParams ( params ) { const getId = ( id ) : number => { let ret = id ; if ( typeof id === "object" && id . hasOwnProperty ( "id" ) ) { ret = id . id ; } return ret ; } ; const paramsForSearch = { ... params } ; if ( paramsForSearch . locationId === undefined ) { paramsForSearch . locationId = 0 ; } if ( paramsForSearch . categoryId === undefined ) { paramsForSearch . categoryId = 0 ; } paramsForSearch . formSubmit = true ; paramsForSearch . siteLocale = "en_CA" ; paramsForSearch . locationId = getId ( paramsForSearch . locationId ) ; paramsForSearch . categoryId = getId ( paramsForSearch . categoryId ) ; ensureIntProp ( paramsForSearch , "locationId" ) ; ensureIntProp ( paramsForSearch , "categoryId" ) ; return paramsForSearch as ResolvedSearchParameters ; } function getSearchOptions ( options ) { const optionsForSearch = { ... options } ; if ( optionsForSearch . pageDelayMs === undefined ) { optionsForSearch . pageDelayMs = 1000 ; } ensureIntProp ( optionsForSearch , "pageDelayMs" ) ; if ( optionsForSearch . scrapeResultDetails === undefined ) { optionsForSearch . scrapeResultDetails = true ; } if ( optionsForSearch . resultDetailsDelayMs === undefined ) { optionsForSearch . resultDetailsDelayMs = 500 ; } ensureIntProp ( optionsForSearch , "resultDetailsDelayMs" ) ; if ( optionsForSearch . maxResults === undefined ) { optionsForSearch . maxResults = - 1 ; } ensureIntProp ( optionsForSearch , "maxResults" ) ; if ( optionsForSearch . minResults === undefined ) { if ( optionsForSearch . maxResults > 0 ) { optionsForSearch . minResults = optionsForSearch . maxResults ; } else { optionsForSearch . minResults = 20 ; } } else if ( optionsForSearch . minResults < 0 ) { optionsForSearch . minResults = optionsForSearch . maxResults ; } ensureIntProp ( optionsForSearch , "minResults" ) ; return optionsForSearch as Required < SearchOptions > ; } param params param options param callback returns export function search ( params , options = { } , callback ? ) { const promise = new Promise ( ( resolve , reject ) => { const paramsForSearch = getSearchParams ( params ) ; const optionsForSearch = getSearchOptions ( options ) ; const scraperOptions = getScraperOptions ( options ) ; const searcher = scraperOptions . scraperType === ScraperType . HTML ? new HTMLSearcher ( ) : new APISearcher ( ) ; getSearchResults ( searcher , paramsForSearch , optionsForSearch ) . then ( async ( results ) => { if ( optionsForSearch . maxResults >= 0 ) { results = results . slice ( 0 , optionsForSearch . maxResults ) ; } if ( optionsForSearch . scrapeResultDetails ) { if ( optionsForSearch . resultDetailsDelayMs > 0 ) { for ( const ad of results ) { if ( ! ad . isScraped ( ) ) { await ad . scrape ( ) ; await sleep ( optionsForSearch . resultDetailsDelayMs ) ; } } } else { await Promise . all ( results . map ( ad => { if ( ! ad . isScraped ( ) ) { ad . scrape ( ) ; } } ) ) ; } } return results ; } ) . then ( resolve , reject ) ; } ) ; if ( callback ) { promise . then ( results => callback ( null , results ) , err => callback ( err , [ ] ) ) ; } return promise ; }