fileoverview import * as url from 'url' ; import { promisify } from 'util' ; import * as zlib from 'zlib' ; import * as request from 'request' ; import * as iconv from 'iconv-lite' ; import parseDataURL = require ( 'data-urls' ) ; import { HttpHeaders } from '@hint/utils-types' ; import { getContentTypeData } from '@hint/utils' ; import { normalizeHeaderValue } from '@hint/utils-network' ; import { debug as d } from '@hint/utils-debug' ; import { toLowerCaseKeys } from '@hint/utils-string' ; import { NetworkData } from 'hint' ; import { RedirectManager } from './redirects' ; interface IDecompressor { ( content ) : Promise < Buffer > ; } const debug = d ( __filename ) ; const decompressBrotli = promisify ( zlib . brotliDecompress ) ; const decompressGzip = promisify ( zlib . gunzip ) ; const inflateAsync = promisify ( zlib . inflate ) ; const inflateRawAsync = promisify ( zlib . inflateRaw ) ; const inflate = ( buff ) : Promise < Buffer > => { if ( ( buff [ 0 ] & 0x0f ) === 8 && ( buff . readUInt16BE ( 0 ) % 31 === 0 ) ) { return inflateAsync ( buff ) as any ; } return inflateRawAsync ( buff ) as any ; } ; const identity = ( buff ) : Promise < Buffer > => { return Promise . resolve ( Buffer . from ( buff ) ) ; } ; const defaults = { encoding : null , followRedirect : false , headers : { 'Accept-Encoding' : 'gzip, deflate, br' , 'Accept-Language' : 'en-US,en;q=0.8,es;q=0.6,fr;q=0.4' , 'Cache-Control' : 'no-cache' , DNT : 1 , Pragma : 'no-cache' , 'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36' } , jar : true , time : true , timeout : 30000 } ; export class Requester { /** The valid status codes for redirects we follow. */ private static validRedirects = [ 301 , 302 , 303 , 307 , 308 ] ; /** Internal `request` object. */ private _request : request . RequestAPI < request . Request , request . CoreOptions , request . RequiredUriUrl > ; /** Internal `redirectManager`. */ private _redirects : RedirectManager = new RedirectManager ( ) ; /** Maximum number of redirects */ private _maxRedirects : number = 10 ; /** Internal options for request */ private _options : request . CoreOptions ; /** Tries to decompress the given `Buffer` `content` using the given `decompressor` */ private async tryToDecompress ( decompressor , content ) { try { const result = await decompressor ( content ) ; return result ; } catch ( e ) { return null ; } } /** Returns the functions to try to use in order for a given algorithm. */ private decompressors ( algorithm ) { const priorities = { br : 0 , gzip : 1 , deflate : 2 , identity : 3 } ; const functions = [ decompressBrotli , decompressGzip , inflate , identity ] ; const priority = typeof priorities [ algorithm . trim ( ) ] === 'undefined' ? priorities . identity : priorities [ algorithm ] ; return functions . slice ( priority ) ; } /**
     * Tries to uncompresses a buffer with fallbacks in case `content-encoding`
     * is not accurate. E.g.:
     * `Content-Encoding` is `br` but content is actually `gzip`. It will try
     * first with Brotli, then gzip, then return a copy of the original Buffer
     *
     */ private async decompressResponse ( contentEncoding , rawBodyResponse ) { const that = this ; const algorithms = contentEncoding ? contentEncoding . split ( ',' ) : [ '' ] ; const decompressors = this . decompressors ( algorithms . shift ( ) ! . trim ( ) ) as IDecompressor [ ] ; let rawBody = null ; for ( const decompressor of decompressors ) { rawBody = await that . tryToDecompress ( decompressor , rawBodyResponse ) ; if ( rawBody ) { break ; } } if ( rawBody && algorithms . length > 0 ) { return this . decompressResponse ( algorithms . join ( ',' ) , rawBody ) ; } return rawBody ; } public constructor ( customOptions ? ) { if ( customOptions ) { customOptions . followRedirect = false ; customOptions . rejectUnauthorized = false ; this . _maxRedirects = customOptions . maxRedirects || this . _maxRedirects ; if ( customOptions . headers ) { customOptions . headers = { ... toLowerCaseKeys ( defaults . headers ) , ... toLowerCaseKeys ( customOptions . headers ) } ; } } const options = { ... defaults , ... customOptions } ; this . _options = options ; this . _request = request . defaults ( options ) ; } /** Return the redirects for a given `uri`. */ public getRedirects ( uri ) { return this . _redirects . calculate ( uri ) ; } private getResourceNetworkDataFromDataUri ( uri ) { const parsedDataURL = parseDataURL ( uri ) ; const networkData = { request : { headers : { } , url : uri } , response : { body : { content : parsedDataURL . body as any , rawContent : parsedDataURL . body , rawResponse : ( ) => { return Promise . resolve ( parsedDataURL . body ) ; } } , charset : parsedDataURL . mimeType . parameters . get ( 'charset' ) || '' , headers : { } , hops : [ ] , mediaType : parsedDataURL . mimeType . toString ( ) , statusCode : 200 , url : uri } } ; return networkData ; } /**
     * Performs a `get` to the given `uri`.
     * If `Content-Type` is of type text and the charset is one of those supported by
     * [`iconv-lite`](https://github.com/ashtuchkin/iconv-lite/wiki/Supported-Encodings)
     * it will decode the response.
     */ public get ( uri ) { debug ( `Requesting ${ uri }` ) ; if ( uri . startsWith ( 'data:' ) ) { return Promise . resolve ( this . getResourceNetworkDataFromDataUri ( uri ) ) ; } const requestedUrls = new Set ( ) ; const getUri = ( uriString ) : Promise < NetworkData > => { requestedUrls . add ( uriString ) ; return new Promise ( ( resolve , reject ) => { const byteChunks = [ ] ; let rawBodyResponse ; this . _request ( { uri : uriString } , async ( err , response ) => { if ( err ) { debug ( `Request for ${ uriString } failed\n${ err }` ) ; return reject ( { error : err , uri : uriString } ) ; } if ( Requester . validRedirects . includes ( response . statusCode ) ) { if ( ! response . headers . location ) { return reject ( { error : new Error ( 'Redirect location undefined' ) , uri : uriString } ) ; } const newUri = url . resolve ( uriString , response . headers . location as string ) ; if ( requestedUrls . has ( newUri ) ) { return reject ( new Error ( `'${ uriString }' could not be fetched using ${ this . _options . method || 'GET' } method (redirect loop detected).` ) ) ; } this . _redirects . add ( newUri , uriString ) ; const currentRedirectNumber = this . _redirects . calculate ( newUri ) . length ; if ( currentRedirectNumber > this . _maxRedirects ) { return reject ( new Error ( `The number of redirects(${ currentRedirectNumber }) exceeds the limit(${ this . _maxRedirects }).` ) ) ; } try { debug ( `Redirect found for ${ uriString }` ) ; const results = await getUri ( newUri ) ; return resolve ( results ) ; } catch ( e ) { return reject ( e ) ; } } const contentEncoding = normalizeHeaderValue ( response . headers as HttpHeaders , 'content-encoding' ) ; const rawBody = await this . decompressResponse ( contentEncoding , rawBodyResponse ) ; const contentTypeData = await getContentTypeData ( null , uri , response . headers as HttpHeaders , rawBody as Buffer ) ; const charset = contentTypeData . charset || '' ; const mediaType = contentTypeData . mediaType || '' ; const hops = this . _redirects . calculate ( uriString ) ; const body = rawBody && iconv . encodingExists ( charset ) ? iconv . decode ( rawBody , charset ) : null ; const networkData = { request : { headers : response . request . headers , url : hops [ 0 ] || uriString } , response : { body : { content : body as string , rawContent : rawBody as Buffer , rawResponse : ( ) => { return Promise . resolve ( rawBodyResponse ) ; } } , charset , headers : response . headers as HttpHeaders , hops , mediaType , statusCode : response . statusCode , url : uriString } } ; return resolve ( networkData ) ; } ) . on ( 'response' , ( response ) => { response . on ( 'data' , ( data ) => { byteChunks . push ( data ) ; } ) . on ( 'end' , ( ) => { rawBodyResponse = Buffer . concat ( byteChunks ) ; } ) ; } ) ; } ) ; } ; return getUri ( uri ) ; } }