import assert = require ( "assert" ) ; import { readdir } from "fs-extra" ; import * as path from "path" ; import { Logger , logger , logPath , writeLog } from "../util/logging" ; import { joinPaths , unique } from "../util/util" ; import BlobWriter , { urlOfBlob } from "./azure-container" ; const maxNumberOfOldLogsDirectories = 5 ; export default async function uploadBlobsAndUpdateIssue ( timeStamp ) { const container = await BlobWriter . create ( ) ; await container . ensureCreated ( { publicAccessLevel : "blob" } ) ; await container . setCorsProperties ( ) ; const [ dataUrls , logUrls ] = await uploadBlobs ( container , timeStamp ) ; await uploadIndex ( container , timeStamp , dataUrls , logUrls ) ; } async function uploadBlobs ( container , timeStamp ) { const [ log , logResult ] = logger ( ) ; const [ dataUrls , logUrls ] = await Promise . all ( [ await uploadDirectory ( container , "data" , "data" , log ) , await uploadLogs ( container , timeStamp , log ) , ] ) ; const blobLogs = "upload-blobs.md" ; await writeLog ( blobLogs , logResult ( ) ) ; logUrls . push ( await uploadFile ( container , `${ logsUploadedLocation ( timeStamp ) }/${ blobLogs }` , logPath ( blobLogs ) ) ) ; return [ dataUrls , logUrls ] ; } const logsDirectoryName = "logs" ; const logsPrefix = `${ logsDirectoryName }/` ; function logsUploadedLocation ( timeStamp ) { return logsPrefix + timeStamp ; } async function uploadLogs ( container , timeStamp , log ) { await removeOldDirectories ( container , logsPrefix , maxNumberOfOldLogsDirectories - 1 , log ) ; return uploadDirectory ( container , logsUploadedLocation ( timeStamp ) , logsDirectoryName , log , f => f !== "upload-blobs.md" ) ; } async function uploadDirectory ( container , uploadedDirPath , dirPath , log , filter ? ) { let files = await readdir ( dirPath ) ; if ( filter ) { files = files . filter ( filter ) ; } return Promise . all ( files . map ( fileName => { const fullPath = joinPaths ( dirPath , fileName ) ; const blobName = joinPaths ( uploadedDirPath , fileName ) ; return logAndUploadFile ( container , blobName , fullPath , log ) ; } ) ) ; } async function logAndUploadFile ( container , blobName , filePath , log ) { const url = urlOfBlob ( blobName ) ; log ( `Uploading ${ filePath } to ${ url }` ) ; await container . createBlobFromFile ( blobName , filePath ) ; return url ; } async function uploadFile ( container , blobName , filePath ) { const url = urlOfBlob ( blobName ) ; await container . createBlobFromFile ( blobName , filePath ) ; return url ; } async function deleteDirectory ( container , uploadedDirPath , log ) { const blobs = await container . listBlobs ( uploadedDirPath ) ; const blobNames = blobs . map ( b => b . name ) ; log ( `Deleting directory ${ uploadedDirPath }: delete files ${ blobNames . toString ( ) }` ) ; await Promise . all ( blobNames . map ( b => container . deleteBlob ( b ) ) ) ; } async function removeOldDirectories ( container , prefix , maxDirectories , log ) { const list = await container . listBlobs ( prefix ) ; const dirNames = unique ( list . map ( ( { name } ) => { assert ( name . startsWith ( prefix ) ) ; return path . dirname ( name . slice ( prefix . length ) ) ; } ) ) ; if ( dirNames . length <= maxDirectories ) { log ( `No need to remove old directories: have ${ dirNames . length }, can go up to ${ maxDirectories }.` ) ; return ; } const sortedNames = dirNames . sort ( ) ; const toDelete = sortedNames . slice ( 0 , sortedNames . length - maxDirectories ) ; log ( `Too many old logs, so removing the following directories: [${ toDelete . toString ( ) }]` ) ; await Promise . all ( toDelete . map ( d => deleteDirectory ( container , prefix + d , log ) ) ) ; } function uploadIndex ( container , timeStamp , dataUrls , logUrls ) { return container . createBlobFromText ( "index.html" , createIndex ( ) ) ; function createIndex ( ) { const lines = [ ] ; lines . push ( "<html><head></head><body>" ) ; lines . push ( `<h3>Here is the latest data as of **${ timeStamp }**:</h3>` ) ; lines . push ( "<h4>Data</h4>" ) ; lines . push ( ... dataUrls . map ( link ) ) ; lines . push ( "<h4>Logs</h4>" ) ; lines . push ( ... logUrls . map ( link ) ) ; lines . push ( "</body></html>" ) ; return lines . join ( "\n" ) ; function link ( url ) { const short = url . slice ( url . lastIndexOf ( "/" ) + 1 ) ; return `<li><a href='${ url }'>${ short }</a></li>` ; } } }