import Fs from 'fs/promises' ; import { createWriteStream } from 'fs' ; import Path from 'path' ; import { promisify } from 'util' ; import { pipeline } from 'stream' ; import { ToolingLog } from '@kbn/dev-utils' ; import Axios from 'axios' ; import del from 'del' ; import AxiosHttpAdapter from 'axios/lib/adapters/http' ; interface Archive { sha : string ; path : string ; time : number ; } const asyncPipeline = promisify ( pipeline ) ; async function getCacheNames ( cacheDir ) { try { return await Fs . readdir ( cacheDir ) ; } catch ( error ) { if ( error . code === 'ENOENT' ) { return [ ] ; } throw error ; } } export class Archives { static async create ( log , workingDir ) { const dir = Path . resolve ( workingDir , 'archives' ) ; const bySha = new Map < string , Archive > ( ) ; for ( const name of await getCacheNames ( dir ) ) { const path = Path . resolve ( dir , name ) ; if ( ! name . endsWith ( '.zip' ) ) { log . debug ( 'deleting unexpected file in archives dir' , path ) ; await Fs . unlink ( path ) ; continue ; } const sha = name . replace ( '.zip' , '' ) ; log . verbose ( 'identified archive for' , sha ) ; const s = await Fs . stat ( path ) ; const time = Math . max ( s . atimeMs , s . mtimeMs ) ; bySha . set ( sha , { path , time , sha , } ) ; } return new Archives ( log , workingDir , bySha ) ; } protected constructor ( private readonly log , private readonly workDir , private readonly bySha ) { } size ( ) { return this . bySha . size ; } get ( sha ) { return this . bySha . get ( sha ) ; } async delete ( sha ) { const archive = this . get ( sha ) ; if ( archive ) { await Fs . unlink ( archive . path ) ; this . bySha . delete ( sha ) ; } } * [ Symbol . iterator ] ( ) { yield * this . bySha . values ( ) ; } param sha async attemptToDownload ( sha ) { if ( this . bySha . has ( sha ) ) { return true ; } const url = `https://ts-refs-cache.kibana.dev/${ sha }.zip` ; this . log . debug ( 'attempting to download cache for' , sha , 'from' , url ) ; const filename = `${ sha }.zip` ; const target = Path . resolve ( this . workDir , 'archives' , `${ filename }` ) ; const tmpTarget = `${ target }.tmp` ; try { const resp = await Axios . request ( { url , responseType : 'stream' , adapter : AxiosHttpAdapter , } ) ; await Fs . mkdir ( Path . dirname ( target ) , { recursive : true } ) ; await asyncPipeline ( resp . data , createWriteStream ( tmpTarget ) ) ; this . log . debug ( 'download complete, renaming tmp' ) ; await Fs . rename ( tmpTarget , target ) ; this . bySha . set ( sha , { sha , path : target , time : Date . now ( ) , } ) ; this . log . debug ( 'download of cache for' , sha , 'complete' ) ; return true ; } catch ( error ) { await del ( tmpTarget , { force : true } ) ; if ( ! error . response ) { this . log . debug ( `failed to download cache, ignoring error:` , error . message ) ; return false ; } if ( error . response . status === 404 ) { return false ; } this . log . debug ( `failed to download cache,` , error . response . status , 'response' ) ; } } param shas async getFirstAvailable ( shas ) { if ( ! shas . length ) { throw new Error ( 'no possible shas to pick archive from' ) ; } for ( const sha of shas ) { let archive = this . bySha . get ( sha ) ; if ( ! archive && ( await this . attemptToDownload ( sha ) ) ) { archive = this . bySha . get ( sha ) ; } if ( archive ) { return archive ; } this . log . debug ( 'no archive available for' , sha ) ; } return undefined ; } }