import { openSync , writeSync , unlinkSync , closeSync } from 'fs' ; import { dirname } from 'path' ; import { setTimeout } from 'timers/promises' ; import chalk from 'chalk' ; import { createHash } from 'crypto' ; import Axios from 'axios' ; import { ToolingLog , isAxiosResponseError } from '@kbn/dev-utils' ; import AxiosHttpAdapter from 'axios/lib/adapters/http' ; import { mkdirp } from './fs' ; function tryUnlink ( path ) { try { unlinkSync ( path ) ; } catch ( error ) { if ( error . code !== 'ENOENT' ) { throw error ; } } } interface DownloadToDiskOptions { log : ToolingLog ; url : string ; destination : string ; shaChecksum : string ; shaAlgorithm : string ; maxAttempts ? : number ; retryDelaySecMultiplier ? : number ; } export async function downloadToDisk ( { log , url , destination , shaChecksum , shaAlgorithm , maxAttempts = 1 , retryDelaySecMultiplier = 5 , } ) { if ( ! shaChecksum ) { throw new Error ( `${ shaAlgorithm } checksum of ${ url } not provided, refusing to download.` ) ; } if ( maxAttempts < 1 ) { throw new Error ( `[maxAttempts=${ maxAttempts }] must be >= 1` ) ; } let attempt = 0 ; while ( true ) { attempt += 1 ; await mkdirp ( dirname ( destination ) ) ; const fileHandle = openSync ( destination , 'w' ) ; let error ; try { log . debug ( `[${ attempt }/${ maxAttempts }] Attempting download of ${ url }` , chalk . dim ( shaAlgorithm ) ) ; const response = await Axios . request ( { url , responseType : 'stream' , adapter : AxiosHttpAdapter , } ) ; if ( response . status !== 200 ) { throw new Error ( `Unexpected status code ${ response . status } when downloading ${ url }` ) ; } const hash = createHash ( shaAlgorithm ) ; await new Promise ( ( resolve , reject ) => { response . data . on ( 'data' , ( chunk ) => { hash . update ( chunk ) ; writeSync ( fileHandle , chunk ) ; } ) ; response . data . on ( 'error' , reject ) ; response . data . on ( 'end' , resolve ) ; } ) ; const downloadedSha = hash . digest ( 'hex' ) ; if ( downloadedSha !== shaChecksum ) { throw new Error ( `Downloaded checksum ${ downloadedSha } does not match the expected ${ shaAlgorithm } checksum.` ) ; } } catch ( _error ) { error = _error ; } finally { closeSync ( fileHandle ) ; } if ( ! error ) { log . debug ( `Downloaded ${ url } and verified checksum` ) ; return ; } log . debug ( `Download failed: ${ error . message }` ) ; log . debug ( `Deleting downloaded data at ${ destination }` ) ; tryUnlink ( destination ) ; if ( attempt < maxAttempts ) { const sec = attempt * retryDelaySecMultiplier ; log . info ( `Retrying in ${ sec } seconds` ) ; await setTimeout ( sec * 1000 ) ; continue ; } throw error ; } } interface DownloadToStringOptions { log : ToolingLog ; url : string ; expectStatus ? : number ; maxAttempts ? : number ; retryDelaySecMultiplier ? : number ; } export async function downloadToString ( { log , url , expectStatus , maxAttempts = 3 , retryDelaySecMultiplier = 5 , } ) { let attempt = 0 ; while ( true ) { try { attempt += 1 ; log . debug ( `[${ attempt }/${ maxAttempts }] Attempting download to string of [${ url }]` ) ; const resp = await Axios . request < string > ( { url , method : 'GET' , adapter : AxiosHttpAdapter , responseType : 'text' , validateStatus : ! expectStatus ? undefined : ( status ) => status === expectStatus , } ) ; log . success ( `Downloaded [${ url }]` ) ; return resp . data ; } catch ( error ) { log . warning ( `Download failed: ${ error . message }` ) ; if ( isAxiosResponseError ( error ) ) { log . debug ( `[${ error . response . status }/${ error . response . statusText }] response: ${ error . response . data }` ) ; } else { log . debug ( 'received no response' ) ; } if ( ( maxAttempts ?? 3 ) > attempt ) { const sec = ( retryDelaySecMultiplier ?? 5 ) * attempt ; log . info ( `Retrying in ${ sec } seconds` ) ; await setTimeout ( sec * 1000 ) ; continue ; } throw error ; } } }