import { Agents } from "./agents" ; import { fetch , isValidProtocol , urlToRequestOptions } from "./utils" ; import { URL } from "url" ; export interface Timestamped { timestamp : string ; } export interface FetchOptions { method : "GET" ; headers : Record < string , string | string [ ] > ; } export async function fetchText ( url , agent ? ) { let parsed = new URL ( url ) ; if ( ! isValidProtocol ( parsed . protocol ) ) { throw new Error ( `Invalid protocol ${ parsed . protocol }` ) ; } let { message : msg } = await fetch ( { method : "GET" , ... urlToRequestOptions ( parsed ) , headers : { "user-agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 Edg/80.0.361.48" , } , } , agent ) ; let buf = await new Promise < Buffer > ( ( resolve , reject ) => { let contents = [ ] ; msg . on ( "data" , ( chunk ) => { contents . push ( chunk ) ; } ) ; msg . on ( "end" , ( ) => { resolve ( Buffer . concat ( contents ) ) ; } ) ; msg . on ( "error" , reject ) ; } ) ; return buf . toString ( ) ; } export async function fetchJson ( url , agent ? ) { return JSON . parse ( await fetchText ( url , agent ) ) ; } export async function getIfUpdate ( url , timestamp ? , agent = { } ) { let parsed = new URL ( url ) ; if ( ! isValidProtocol ( parsed . protocol ) ) { throw new Error ( `Invalid protocol ${ parsed . protocol }` ) ; } let { message : msg } = await fetch ( { method : "GET" , ... urlToRequestOptions ( parsed ) , headers : { "user-agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 Edg/80.0.361.48" , "If-Modified-Since" : timestamp ?? "" , } , } , agent ) ; let buf = await new Promise < Buffer > ( ( resolve , reject ) => { let contents = [ ] ; msg . on ( "data" , ( chunk ) => { contents . push ( chunk ) ; } ) ; msg . on ( "end" , ( ) => { resolve ( Buffer . concat ( contents ) ) ; } ) ; msg . on ( "error" , reject ) ; } ) ; let { statusCode , headers } = msg ; if ( statusCode === 304 ) { return { timestamp : headers [ "last-modified" ] ! , content : undefined , } ; } else if ( statusCode === 200 || statusCode === 204 ) { return { timestamp : headers [ "last-modified" ] ! , content : buf . toString ( ) , } ; } throw new Error ( `Failure on response status code: ${ statusCode }.` ) ; } export async function getAndParseIfUpdate < T extends Timestamped > ( url , parser , lastObject ) { let { content , timestamp } = await getIfUpdate ( url , lastObject ?. timestamp ) ; if ( content ) { return { ... parser ( content ) , timestamp , } ; } return lastObject ! ; } export async function getLastModified ( url , timestamp , agent = { } ) { let parsed = new URL ( url ) ; if ( ! isValidProtocol ( parsed . protocol ) ) { throw new Error ( `Invalid protocol ${ parsed . protocol }` ) ; } let { message : msg } = await fetch ( { method : "HEAD" , ... urlToRequestOptions ( parsed ) , headers : { "user-agent" : "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36 Edg/80.0.361.48" , "If-Modified-Since" : timestamp ?? "" , } , } , agent ) ; msg . resume ( ) ; let { headers , statusCode } = msg ; if ( statusCode === 304 ) { return [ true , headers [ "last-modified" ] ] as const ; } else if ( statusCode === 200 || statusCode === 204 ) { return [ false , headers [ "last-modified" ] ] as const ; } throw new Error ( `Failure on response status code: ${ statusCode }.` ) ; }